Update and re-run generation scripts

- Update comments to point to the correct files
- bin/generate_digits_characters was missing

Author: Jacopo Beschi

bin/generate_digits_characters

@@ -0,0 +1,34 @@
+#!/usr/bin/env ruby
+
+require "net/http"
+require "csv"
+
+# Retrieve all the Unicode digits grouped by script and stores them in a .csv file.
+
+puts "loading..."
+
+unicode_util_uri = URI("https://util.unicode.org/UnicodeJsps/list-unicodeset.jsp?a=%5B%3Adigit%3A%5D&g=&i=")
+html = Net::HTTP.get(unicode_util_uri)
+
+# Based on the response HTML.
+CHARACTER_OR_REPETITION_REGEXP = %r{u\+([a-z0-9]+)\</a\>\</code\>\</td\>\<td\>([a-z\- ]+)}i
+digits_and_scripts = html.scan(CHARACTER_OR_REPETITION_REGEXP)
+
+digits_and_scripts.map! do |digit, script|
+  # Rely on String.undump to convert unicode codepoints to utf8 characters.
+  unicode_digit = digit.to_i(16).chr("utf-8")
+  script_name = script.split(" ").tap(&:pop).join("_").downcase
+  [ unicode_digit, script_name ]
+end
+
+puts "processed #{digits_and_scripts.length} characters"
+
+csv = CSV.generate do |csv|
+  digits_and_scripts.each do |digit, script|
+    csv << [ digit, script ]
+  end
+end
+
+File.write(File.join(__dir__, "../", "lib/homographic_spoofing/detector/rule/data/digits.csv"), csv)
+
+puts "done"

lib/homographic_spoofing/detector/rule/data/allowed_idn_characters.txt

@@ -388,6 +388,16 @@
 𐴷,hanifi_rohingya_digit
 𐴸,hanifi_rohingya_digit
 𐴹,hanifi_rohingya_digit
+𐵀,garay_digit
+𐵁,garay_digit
+𐵂,garay_digit
+𐵃,garay_digit
+𐵄,garay_digit
+𐵅,garay_digit
+𐵆,garay_digit
+𐵇,garay_digit
+𐵈,garay_digit
+𐵉,garay_digit
 𑁦,brahmi_digit
 𑁧,brahmi_digit
 𑁨,brahmi_digit
@@ -478,6 +488,26 @@
 𑛇,takri_digit
 𑛈,takri_digit
 𑛉,takri_digit
+𑛐,myanmar_pao_digit
+𑛑,myanmar_pao_digit
+𑛒,myanmar_pao_digit
+𑛓,myanmar_pao_digit
+𑛔,myanmar_pao_digit
+𑛕,myanmar_pao_digit
+𑛖,myanmar_pao_digit
+𑛗,myanmar_pao_digit
+𑛘,myanmar_pao_digit
+𑛙,myanmar_pao_digit
+𑛚,myanmar_eastern_pwo_karen_digit
+𑛛,myanmar_eastern_pwo_karen_digit
+𑛜,myanmar_eastern_pwo_karen_digit
+𑛝,myanmar_eastern_pwo_karen_digit
+𑛞,myanmar_eastern_pwo_karen_digit
+𑛟,myanmar_eastern_pwo_karen_digit
+𑛠,myanmar_eastern_pwo_karen_digit
+𑛡,myanmar_eastern_pwo_karen_digit
+𑛢,myanmar_eastern_pwo_karen_digit
+𑛣,myanmar_eastern_pwo_karen_digit
 𑜰,ahom_digit
 𑜱,ahom_digit
 𑜲,ahom_digit
@@ -508,6 +538,16 @@
 𑥗,dives_akuru_digit
 𑥘,dives_akuru_digit
 𑥙,dives_akuru_digit
+𑯰,sunuwar_digit
+𑯱,sunuwar_digit
+𑯲,sunuwar_digit
+𑯳,sunuwar_digit
+𑯴,sunuwar_digit
+𑯵,sunuwar_digit
+𑯶,sunuwar_digit
+𑯷,sunuwar_digit
+𑯸,sunuwar_digit
+𑯹,sunuwar_digit
 𑱐,bhaiksuki_digit
 𑱑,bhaiksuki_digit
 𑱒,bhaiksuki_digit
@@ -548,6 +588,16 @@
 𑽗,kawi_digit
 𑽘,kawi_digit
 𑽙,kawi_digit
+𖄰,gurung_khema_digit
+𖄱,gurung_khema_digit
+𖄲,gurung_khema_digit
+𖄳,gurung_khema_digit
+𖄴,gurung_khema_digit
+𖄵,gurung_khema_digit
+𖄶,gurung_khema_digit
+𖄷,gurung_khema_digit
+𖄸,gurung_khema_digit
+𖄹,gurung_khema_digit
 𖩠,mro_digit
 𖩡,mro_digit
 𖩢,mro_digit
@@ -578,6 +628,26 @@
 𖭗,pahawh_hmong_digit
 𖭘,pahawh_hmong_digit
 𖭙,pahawh_hmong_digit
+𖵰,kirat_rai_digit
+𖵱,kirat_rai_digit
+𖵲,kirat_rai_digit
+𖵳,kirat_rai_digit
+𖵴,kirat_rai_digit
+𖵵,kirat_rai_digit
+𖵶,kirat_rai_digit
+𖵷,kirat_rai_digit
+𖵸,kirat_rai_digit
+𖵹,kirat_rai_digit
+𜳰,outlined_digit
+𜳱,outlined_digit
+𜳲,outlined_digit
+𜳳,outlined_digit
+𜳴,outlined_digit
+𜳵,outlined_digit
+𜳶,outlined_digit
+𜳷,outlined_digit
+𜳸,outlined_digit
+𜳹,outlined_digit
 𝟎,mathematical_bold_digit
 𝟏,mathematical_bold_digit
 𝟐,mathematical_bold_digit
@@ -658,6 +728,16 @@
 𞓷,nag_mundari_digit
 𞓸,nag_mundari_digit
 𞓹,nag_mundari_digit
+𞗱,ol_onal_digit
+𞗲,ol_onal_digit
+𞗳,ol_onal_digit
+𞗴,ol_onal_digit
+𞗵,ol_onal_digit
+𞗶,ol_onal_digit
+𞗷,ol_onal_digit
+𞗸,ol_onal_digit
+𞗹,ol_onal_digit
+𞗺,ol_onal_digit
 𞥐,adlam_digit
 𞥑,adlam_digit
 𞥒,adlam_digit

lib/homographic_spoofing/detector/rule/data/digits.csv

@@ -128,7 +128,7 @@ def allowed_chars_set
     end
 
     private
-      # Built with script/development/generate_allowed_idn_characters.rb
+      # Built with bin/generate_allowed_idn_characters
       def read_allowed_idn_chars
         File.read("#{__dir__}/data/allowed_idn_characters.txt")
       end

lib/homographic_spoofing/detector/rule/disallowed_characters.rb

@@ -23,7 +23,7 @@ def build_digits_map
       end
     end
 
-    # Built with script/development/generate_digits_characters.rb
+    # Built with bin/generate_digits_characters
     def read_digits
       File.read("#{__dir__}/data/digits.csv")
     end