Add logging for silent failures, narrow constraint rescues, and improve test coverage

- Add warn/verbose logging to eliminate silent data loss cascade:
  country_table, state_table, county_table, and zipcode_table now log
  when rows are skipped due to missing lookups
- Add warn_once helper to DbTable for deduplicated warnings
- Narrow rescue SQLite3::ConstraintException to only swallow UNIQUE
  violations; re-raise NOT NULL, FOREIGN KEY, and CHECK violations
- Extract find_state_where helper to keep get_state_id under method
  length limit
- Add early return for nil/empty country in get_state_id
- Expand synthesize_state comment to document Kiba row-hash mutation
- Add missing tests: zipcode state_id nil guard, state country_id nil
  guard, and cross-country integration tests through full Kiba pipeline
- Silence logger and initialize Options in spec_helper for all tests

Author: midwire

lib/free_zipcode_data/country_table.rb

@@ -25,7 +25,10 @@ def build
 
     def write(row)
       country_hash = country_lookup_table[row[:country]]
-      return update_progress unless country_hash
+      unless country_hash
+        warn_once("Skipping unknown country '#{row[:country]}': not in country_lookup_table")
+        return update_progress
+      end
 
       sql = <<-SQL
         INSERT INTO countries (alpha2, alpha3, iso, name)
@@ -37,8 +40,10 @@ def write(row)
 
       begin
         database.execute(sql)
-      rescue SQLite3::ConstraintException
-        # Swallow duplicates
+      rescue SQLite3::ConstraintException => e
+        unless e.message.include?('UNIQUE')
+          raise "Please file an issue at #{ISSUE_URL}: [#{e}] -> SQL: [#{sql}]"
+        end
       rescue StandardError => e
         raise "Please file an issue at #{ISSUE_URL}: [#{e}] -> SQL: [#{sql}]"
       end

lib/free_zipcode_data/county_table.rb

@@ -27,7 +27,13 @@ def write(row)
       return nil unless row[:county]
 
       state_id = get_state_id(row[:country], row[:short_state], row[:state])
-      return nil unless state_id
+      unless state_id
+        logger.verbose(
+          "Skipping county '#{row[:county]}': no state found for " \
+          "abbr='#{row[:short_state]}', country='#{row[:country]}'"
+        )
+        return nil
+      end
 
       sql = <<-SQL
         INSERT INTO counties (state_id, abbr, name)
@@ -39,8 +45,10 @@ def write(row)
 
       begin
         database.execute(sql)
-      rescue SQLite3::ConstraintException
-        # swallow duplicates
+      rescue SQLite3::ConstraintException => e
+        unless e.message.include?('UNIQUE')
+          raise "Please file an issue at #{ISSUE_URL}: [#{e}] -> SQL: [#{sql}]"
+        end
       rescue StandardError => e
         raise "Please file an issue at #{ISSUE_URL}: [#{e}] -> SQL: [#{sql}]"
       end

lib/free_zipcode_data/db_table.rb

@@ -24,6 +24,18 @@ def update_progress
 
     private
 
+    def logger
+      Logger.instance
+    end
+
+    def warn_once(message)
+      @warned_messages ||= {}
+      return if @warned_messages[message]
+
+      logger.warn(message)
+      @warned_messages[message] = true
+    end
+
     def country_lookup_table
       @country_lookup_table ||=
         begin
@@ -44,44 +56,43 @@ def get_country_id(country)
       select_first(sql)
     end
 
+    # Look up a state ID scoped to a country, trying progressively less specific
+    # criteria: (1) abbr + name + country, (2) abbr + country, (3) name + country.
+    # Returns nil if no match is found.
     def get_state_id(country, state_abbr, state_name)
       escaped_country = escape_single_quotes(country)
+      return nil if escaped_country.empty?
+
       escaped_abbr = escape_single_quotes(state_abbr)
       escaped_name = escape_single_quotes(state_name)
-
-      # Try exact match: abbr + name + country
-      sql = <<-SQL
-        SELECT s.id FROM states s
-        INNER JOIN countries c ON s.country_id = c.id
-        WHERE s.abbr = '#{escaped_abbr}'
-        AND s.name = '#{escaped_name}'
-        AND c.alpha2 = '#{escaped_country}'
-      SQL
-      res = select_first(sql)
+      country_cond = "c.alpha2 = '#{escaped_country}'"
+      # Most specific lookup: abbr + name + country
+      res = find_state_where("s.abbr = '#{escaped_abbr}'", "s.name = '#{escaped_name}'", country_cond)
+      return res if res
 
       # Fallback: abbr + country only
-      if res.nil?
-        sql = <<-SQL
-          SELECT s.id FROM states s
-          INNER JOIN countries c ON s.country_id = c.id
-          WHERE s.abbr = '#{escaped_abbr}'
-          AND c.alpha2 = '#{escaped_country}'
-        SQL
-        res = select_first(sql)
+      res = find_state_where("s.abbr = '#{escaped_abbr}'", country_cond)
+      if res
+        logger.verbose("State fallback: abbr '#{state_abbr}' + country '#{country}' (name mismatch)")
+        return res
       end
-
       # Fallback: name + country only
-      if res.nil?
-        sql = <<-SQL
-          SELECT s.id FROM states s
-          INNER JOIN countries c ON s.country_id = c.id
-          WHERE s.name = '#{escaped_name}'
-          AND c.alpha2 = '#{escaped_country}'
-        SQL
-        res = select_first(sql)
+      res = find_state_where("s.name = '#{escaped_name}'", country_cond)
+      if res
+        logger.verbose("State fallback: name '#{state_name}' + country '#{country}' (abbr mismatch)")
+        return res
       end
+      logger.warn("State lookup failed: abbr='#{state_abbr}', name='#{state_name}', country='#{country}'")
+      nil
+    end
 
-      res
+    def find_state_where(*conditions)
+      sql = <<-SQL
+        SELECT s.id FROM states s
+        INNER JOIN countries c ON s.country_id = c.id
+        WHERE #{conditions.join(' AND ')}
+      SQL
+      select_first(sql)
     end
 
     def get_county_id(county)

lib/free_zipcode_data/state_table.rb

@@ -33,7 +33,10 @@ def write(row)
 
       row[:state] = 'Marshall Islands' if row[:short_state] == 'MH' && row[:state].nil?
       country_id = get_country_id(row[:country])
-      return nil unless country_id
+      unless country_id
+        warn_once("Country '#{row[:country]}' not found in countries table, skipping state")
+        return nil
+      end
 
       sql = <<-SQL
         INSERT INTO states (abbr, name, country_id)
@@ -44,8 +47,10 @@ def write(row)
       SQL
       begin
         database.execute(sql)
-      rescue SQLite3::ConstraintException
-        # Swallow duplicates
+      rescue SQLite3::ConstraintException => e
+        unless e.message.include?('UNIQUE')
+          raise "Please file an issue at #{ISSUE_URL}: [#{e}] -> SQL: [#{sql}]"
+        end
       rescue StandardError => e
         raise "Please file an issue at #{ISSUE_URL}: [#{e}] -> SQL: [#{sql}]"
       end
@@ -55,11 +60,19 @@ def write(row)
 
     private
 
-    # Synthesize state from country for stateless countries (downstream tables need this)
+    # Synthesize state from country for stateless countries.
+    # Mutates the row hash so downstream Kiba destinations (CountyTable, ZipcodeTable)
+    # see the synthesized short_state and state values.
     def synthesize_state(row)
       if row[:short_state].nil? || row[:short_state] == ''
         country_entry = country_lookup_table[row[:country]]
-        return false unless country_entry
+        unless country_entry
+          warn_once(
+            "Cannot synthesize state for country '#{row[:country]}': " \
+            'not in country_lookup_table'
+          )
+          return false
+        end
 
         row[:short_state] = row[:country]
         row[:state] = country_entry[:name]

lib/free_zipcode_data/zipcode_table.rb

@@ -30,7 +30,13 @@ def write(row)
       return nil unless row[:postal_code]
 
       state_id = get_state_id(row[:country], row[:short_state], row[:state])
-      return nil unless state_id
+      unless state_id
+        logger.verbose(
+          "Skipping zipcode '#{row[:postal_code]}': no state found for " \
+          "abbr='#{row[:short_state]}', country='#{row[:country]}'"
+        )
+        return nil
+      end
 
       city_name = escape_single_quotes(row[:city])
 
@@ -47,8 +53,10 @@ def write(row)
 
       begin
         database.execute(sql)
-      rescue SQLite3::ConstraintException => _e
-        # there are some duplicates - swallow them
+      rescue SQLite3::ConstraintException => e
+        unless e.message.include?('UNIQUE')
+          raise "Please file an issue at #{ISSUE_URL}: [#{e}] -> SQL: [#{sql}]"
+        end
       rescue StandardError => e
         raise "Please file an issue at #{ISSUE_URL}: [#{e}] -> SQL: [#{sql}]"
       end

spec/etl/csv_source_spec.rb

@@ -28,7 +28,7 @@
       rows = []
       source.each { |row| rows << row }
 
-      expect(rows.length).to eq(5)
+      expect(rows.length).to eq(6)
       expect(rows.first).to be_a(Hash)
       expect(rows.first.keys).to include(:country, :postal_code, :city)
     end

spec/etl/free_zipcode_data_job_spec.rb

@@ -4,7 +4,7 @@
 require 'etl/free_zipcode_data_job'
 
 RSpec.describe ETL::FreeZipcodeDataJob do
-  let(:db) { create_test_database(line_count: 5) }
+  let(:db) { create_test_database(line_count: 6) }
   let(:fixture_csv) { File.join(FreeZipcodeData.root, 'spec', 'fixtures', 'test_data.csv') }
   let(:logger) { FreeZipcodeData::Logger.instance }
   let(:string_io) { StringIO.new }
@@ -92,5 +92,44 @@
       expect(lat).to be_within(0.01).of(40.7484)
       expect(lon).to be_within(0.01).of(-73.9967)
     end
+
+    it 'scopes duplicate state abbreviations by country' do
+      us_ny = db.execute(<<-SQL)
+        SELECT s.id, s.name, c.alpha2
+        FROM states s
+        JOIN countries c ON s.country_id = c.id
+        WHERE s.abbr = 'NY' AND c.alpha2 = 'US'
+      SQL
+      ca_ny = db.execute(<<-SQL)
+        SELECT s.id, s.name, c.alpha2
+        FROM states s
+        JOIN countries c ON s.country_id = c.id
+        WHERE s.abbr = 'NY' AND c.alpha2 = 'CA'
+      SQL
+      expect(us_ny.length).to eq(1)
+      expect(ca_ny.length).to eq(1)
+      expect(us_ny[0][0]).not_to eq(ca_ny[0][0])
+      expect(us_ny[0][1]).to eq('New York')
+      expect(ca_ny[0][1]).to eq('Northern York')
+    end
+
+    it 'links cross-country zipcodes to the correct state' do
+      us_zip = db.execute(<<-SQL)
+        SELECT z.code, s.name, c.alpha2
+        FROM zipcodes z
+        JOIN states s ON CAST(z.state_id AS INTEGER) = s.id
+        JOIN countries c ON s.country_id = c.id
+        WHERE z.code = '10001'
+      SQL
+      ca_zip = db.execute(<<-SQL)
+        SELECT z.code, s.name, c.alpha2
+        FROM zipcodes z
+        JOIN states s ON CAST(z.state_id AS INTEGER) = s.id
+        JOIN countries c ON s.country_id = c.id
+        WHERE z.code = 'K0A'
+      SQL
+      expect(us_zip[0]).to eq(['10001', 'New York', 'US'])
+      expect(ca_zip[0]).to eq(['K0A', 'Northern York', 'CA'])
+    end
   end
 end

spec/fixtures/test_data.csv

@@ -3,4 +3,5 @@ US,10001,New York,New York,NY,New York,061,Manhattan,MN,40.7484,-73.9967,4
 US,90210,Beverly Hills,California,CA,Los Angeles,037,,LA,34.0901,-118.4065,4
 US,60601,Chicago,Illinois,IL,Cook,031,,CK,41.8819,-87.6278,4
 CA,H2X,Montreal,Quebec,QC,,,Montreal,,45.5088,-73.5878,4
+CA,K0A,Almonte,Northern York,NY,Lanark,LNK,,,45.2260,-76.1840,4
 GB,SW1A,London,England,ENG,Westminster,,City of Westminster,,51.5014,-0.1419,1

spec/free_zipcode_data/state_table_spec.rb

@@ -75,6 +75,13 @@
       expect(rows[0][0]).to eq(0)
     end
 
+    it 'returns nil when country is not in the countries table' do
+      result = table.write({ country: 'DE', short_state: 'BY', state: 'Bavaria' })
+      expect(result).to be_nil
+      rows = db.execute('SELECT COUNT(*) FROM states')
+      expect(rows[0][0]).to eq(0)
+    end
+
     it 'silently ignores duplicate state entries' do
       table.write({ country: 'US', short_state: 'NY', state: 'New York' })
       expect { table.write({ country: 'US', short_state: 'NY', state: 'New York' }) }.not_to raise_error

spec/free_zipcode_data/zipcode_table_spec.rb

@@ -72,6 +72,13 @@
       expect(rows[0][0]).to eq(0)
     end
 
+    it 'returns nil and skips when state cannot be found' do
+      result = table.write(row.merge(short_state: 'ZZ', state: 'Nonexistent'))
+      expect(result).to be_nil
+      rows = db.execute('SELECT COUNT(*) FROM zipcodes')
+      expect(rows[0][0]).to eq(0)
+    end
+
     it 'silently ignores duplicate zipcode entries' do
       table.write(row)
       expect { table.write(row) }.not_to raise_error