fix: improve seeder (and remove symbol dupes)

database/seeders/MarketDataSeeder.php

@@ -12,6 +12,8 @@ class MarketDataSeeder extends Seeder
 {
     use WithoutModelEvents;
 
+    public array $rows = [];
+
     /**
      * Run the database seeds.
      */
@@ -26,7 +28,6 @@ class MarketDataSeeder extends Seeder
         if (($handle = fopen($csvFilePath, 'r')) !== false) {
 
             $header = null;
-            $rows = [];
             $rowCount = 0;
 
             while (($row = fgetcsv($handle, 0, ',')) !== false) {
@@ -38,10 +39,9 @@ class MarketDataSeeder extends Seeder
 
                 } else {
 
-                    try {
                     $data = array_combine($header, $row);
 
-                        $rows[] = [
+                    $this->rows[] = [
                         'symbol' => $data['symbol'],
                         'name' => $data['name'],
                         'meta_data' => json_encode([
@@ -55,19 +55,17 @@ class MarketDataSeeder extends Seeder
                     $rowCount++;
 
                     if ($rowCount % $chunkSize == 0) {
-                            DB::table('market_data')->insertOrIgnore($rows);
-                            $rows = [];
-                        }
-                    } catch (\Throwable $e) {
 
-                        throw new \Exception('Error: '.$e->getMessage());
+                        $this->bulkInsert($this->rows);
                     }
                 }
             }
 
             // final clean up
-            if (! empty($rows)) {
+            if (! empty($this->rows)) {
-                DB::table('market_data')->insertOrIgnore($rows);
+                echo 'did '.$rowCount.' already. doing '.count($this->rows).' more';
+                $this->bulkInsert($this->rows);
+
             }
 
             // Close the CSV file
@@ -80,4 +78,17 @@ class MarketDataSeeder extends Seeder
             echo "Failed to open the CSV.\n";
         }
     }
+
+    public function bulkInsert(array $rows)
+    {
+        try {
+
+            DB::table('market_data')->insertOrIgnore($rows);
+            $this->rows = [];
+
+        } catch (\Throwable $e) {
+
+            throw new \Exception('Error: '.$e->getMessage());
+        }
+    }
 }

database/seeders/market_data_seed.csv

@@ -241,8 +241,6 @@ AKA,a.k.a. Brands Holding Corp. Common Stock,,2021,Consumer Discretionary,Catalo
 AKAM,Akamai Technologies Inc. Common Stock,United States,1999,Consumer Discretionary,Business Services
 AKAN,Akanda Corp. Common Shares,United Kingdom,2022,Health Care, Medicinal Chemicals and Botanical Products 
 AKBA,Akebia Therapeutics Inc. Common Stock,United States,2014,Health Care,Biotechnology: Pharmaceutical Preparations
-AKO-A,Embotelladora Andina S.A.,Chile,,,
-AKO-B,Embotelladora Andina S.A.,Chile,,,
 AKR,Acadia Realty Trust Common Stock,United States,,Real Estate,Real Estate Investment Trusts
 AKRO,Akero Therapeutics Inc. Common Stock,United States,2019,Health Care,Biotechnology: Pharmaceutical Preparations
 AKTS,Akoustis Technologies Inc. Common Stock,United States,,Utilities,Telecommunications Equipment
@@ -789,8 +787,6 @@ BERY,Berry Global Group Inc. Common Stock,United States,2012,Industrials,Plastic
 BEST,BEST Inc. American Depositary Shares each representing twenty (20) Class A Ordinary Shares,Cayman Islands,2023,Industrials,Trucking Freight/Courier Services
 BETR,Better Home & Finance Holding Company Class A Common Stock,United Kingdom,2021,Finance,Finance: Consumer Services
 BETRW,Better Home & Finance Holding Company Warrant,United Kingdom,2021,Finance,Finance: Consumer Services
-BF-A,Brown Forman Corporation,United States,,,
-BF-B,Brown Forman Corporation,United States,,,
 BFAC,Battery Future Acquisition Corp. Class A Ordinary Shares,,2022,Finance,Blank Checks
 BFAM,Bright Horizons Family Solutions Inc. Common Stock,United States,2013,Consumer Discretionary,Other Consumer Services
 BFC,Bank First Corporation Common Stock,United States,,Finance,Major Banks
@@ -1004,8 +1000,6 @@ BREA,Brera Holdings PLC Class B Ordinary Shares,Ireland,2023,Consumer Discretion
 BRFH,Barfresh Food Group Inc. Common Stock,United States,,Consumer Staples,Packaged Foods
 BRFS,BRF S.A.,Brazil,,Consumer Staples,Meat/Poultry/Fish
 BRID,Bridgford Foods Corporation Common Stock,United States,,Consumer Staples,Specialty Foods
-BRK-A,Berkshire Hathaway Inc.,United States,,,
-BRK-B,Berkshire Hathaway Inc.,United States,,,
 BRKH,BurTech Acquisition Corp. Class A Common Stock,United States,2022,Finance,Blank Checks
 BRKHU,BurTech Acquisition Corp. Unit,United States,2021,Finance,Blank Checks
 BRKHW,BurTech Acquisition Corp. Warrants,United States,2022,Finance,Blank Checks
@@ -1542,8 +1536,6 @@ CRBP,Corbus Pharmaceuticals Holdings Inc. Common Stock,United States,,Health Car
 CRBU,Caribou Biosciences Inc. Common Stock,United States,2021,Health Care,Biotechnology: Biological Products (No Diagnostic Substances)
 CRC,California Resources Corporation Common Stock,United States,2020,Energy,Oil & Gas Production
 CRCT,Cricut Inc. Class A Common Stock,United States,2021,Technology,Industrial Machinery/Components
-CRD-A,Crawford & Company,United States,,,
-CRD-B,Crawford & Company,United States,,,
 CRDF,Cardiff Oncology Inc. Common Stock,United States,,Health Care,Biotechnology: Biological Products (No Diagnostic Substances)
 CRDL,Cardiol Therapeutics Inc. Class A Common Shares,Canada,,Health Care,Biotechnology: Biological Products (No Diagnostic Substances)
 CRDO,Credo Technology Group Holding Ltd Ordinary Shares,United States,2022,Technology,Semiconductors
@@ -2850,7 +2842,6 @@ HE,Hawaiian Electric Industries Inc. Common Stock,United States,,Utilities,Elect
 HEAR,Turtle Beach Corporation Common Stock,United States,,Telecommunications,Telecommunications Equipment
 HEES,H&E Equipment Services Inc. Common Stock,United States,2006,Industrials,Misc Corporate Leasing Services
 HEI,Heico Corporation Common Stock,United States,2000,Industrials,Aerospace
-HEI-A,Heico Corporation,United States,,,
 HELE,Helen of Troy Limited Common Stock,Bermuda,,Consumer Discretionary,Home Furnishings
 HEPA,Hepion Pharmaceuticals Inc. Common Stock,United States,,Health Care,Biotechnology: Pharmaceutical Preparations
 HEPS,D-Market Electronic Services & Trading American Depositary Shares,Turkey,2021,Consumer Discretionary,Catalog/Specialty Distribution
@@ -3011,7 +3002,6 @@ HUnited States,Houston American Energy Corporation Common Stock,United States,,E
 HUT,Hut 8 Corp. Common Stock,United States,,Finance,Finance: Consumer Services
 HUYA,HUYA Inc. American depositary shares each representing one Class A ordinary share,United States,2018,Technology,Computer Software: Programming Data Processing
 HVT,Haverty Furniture Companies Inc. Common Stock,United States,,Consumer Discretionary,Other Specialty Stores
-HVT-A,Haverty Furniture Companies Inc.,United States,,,
 HWBK,Hawthorn Bancshares Inc. Common Stock,United States,,Finance,Major Banks
 HWC,Hancock Whitney Corporation Common Stock,United States,,Finance,Major Banks
 HWCPZ,Hancock Whitney Corporation 6.25% Subordinated Notes due 2060,United States,,Finance,Major Banks
@@ -11488,7 +11478,6 @@ NYNYR,Empire Resorts Inc.,United States,,,
 LLEX,"Lilis Energy, Inc.",United States,,Independent Oil & Gas,
 EZT,"Entergy Texas, Inc.",United States,,,
 EXPN.L,Experian plc,United Kingdom,,Business Services,
-ETX,Eaton Vance Municipal Income 2028 Term Trust,United States,,,
 EMI,Eaton Vance Michigan Municipal Income Trust,United States,,,
 EMG,"Emergent Capital, Inc.",United States,,,
 EMCF,Emclaire Financial Corp.,United States,,Regional - Northeast Banks,
@@ -11531,7 +11520,6 @@ ENZY,Enzymotec Ltd.,United States,,Biotechnology,
 ENBP,ENB Financial Corp,United States,,Regional - Northeast Banks,
 EMP-A.TO,Empire Company Limited,Canada,,Gold,
 ECCA,Eagle Point Credit Company Inc.,United States,,,
-ECC,Eagle Point Credit Company Inc.,United States,,,
 NESV,"National Energy Services, Inc.",United States,,,
 YECO,Yulong Eco-Materials Limited,United States,,,
 TIK,Tel-Instrument Electronics Corp.,United States,,Aerospace/Defense - Major Diversified,
@@ -13136,7 +13124,6 @@ HRG,"HRG Group, Inc.",United States,,Conglomerates,
 GSS,Golden Star Resources Ltd.,United States,,Gold,
 GIG,"GigPeak, Inc.",United States,,,
 SGEN,"Seattle Genetics, Inc.",United States,,Biotechnology,
-SAND,Sandstorm Gold Ltd.,United States,,Gold,
 PGEM,"Ply Gem Holdings, Inc",United States,,General Building Materials,
 ININ,"Interactive Intelligence Group, Inc.",United States,,,
 SNR,New Senior Investment Group Inc.,United States,,,
@@ -22537,7 +22524,6 @@ WWSG,Worldwide Strategies Incorporated,United States,,Diversified Communication
 WWTH,"With, Inc.",United States,,,
 WUMSF,Wumart Stores Inc.,United States,,,
 WSSH,West Shore Bank Corp.,United States,,,
-WSO-B,"Watsco, Inc.",United States,,,
 WRCKF,WESC AB,United States,,,
 WMSI,"Williams Industries, Incorporated",United States,,,
 WLKR,Walker Innovation Inc.,United States,,,