From 00fbdec6f16ad52cc0d9de4e851a01a0e8a34487 Mon Sep 17 00:00:00 2001 From: hackerESQ Date: Fri, 7 Mar 2025 18:43:55 -0600 Subject: [PATCH] fix: improve seeder (and remove symbol dupes) --- database/seeders/MarketDataSeeder.php | 55 ++++++++++++++++----------- database/seeders/market_data_seed.csv | 14 ------- 2 files changed, 33 insertions(+), 36 deletions(-) diff --git a/database/seeders/MarketDataSeeder.php b/database/seeders/MarketDataSeeder.php index c6bf500..6217109 100644 --- a/database/seeders/MarketDataSeeder.php +++ b/database/seeders/MarketDataSeeder.php @@ -12,6 +12,8 @@ class MarketDataSeeder extends Seeder { use WithoutModelEvents; + public array $rows = []; + /** * Run the database seeds. */ @@ -26,7 +28,6 @@ class MarketDataSeeder extends Seeder if (($handle = fopen($csvFilePath, 'r')) !== false) { $header = null; - $rows = []; $rowCount = 0; while (($row = fgetcsv($handle, 0, ',')) !== false) { @@ -38,36 +39,33 @@ class MarketDataSeeder extends Seeder } else { - try { - $data = array_combine($header, $row); + $data = array_combine($header, $row); - $rows[] = [ - 'symbol' => $data['symbol'], - 'name' => $data['name'], - 'meta_data' => json_encode([ - 'country' => $data['country'], - 'first_trade_year' => $data['first_trade_year'], - 'sector' => $data['sector'], - 'industry' => $data['industry'], - ]), - ]; + $this->rows[] = [ + 'symbol' => $data['symbol'], + 'name' => $data['name'], + 'meta_data' => json_encode([ + 'country' => $data['country'], + 'first_trade_year' => $data['first_trade_year'], + 'sector' => $data['sector'], + 'industry' => $data['industry'], + ]), + ]; - $rowCount++; + $rowCount++; - if ($rowCount % $chunkSize == 0) { - DB::table('market_data')->insertOrIgnore($rows); - $rows = []; - } - } catch (\Throwable $e) { + if ($rowCount % $chunkSize == 0) { - throw new \Exception('Error: '.$e->getMessage()); + $this->bulkInsert($this->rows); } } } // final clean up - if (! empty($rows)) { - DB::table('market_data')->insertOrIgnore($rows); + if (! empty($this->rows)) { + echo 'did '.$rowCount.' already. doing '.count($this->rows).' more'; + $this->bulkInsert($this->rows); + } // Close the CSV file @@ -80,4 +78,17 @@ class MarketDataSeeder extends Seeder echo "Failed to open the CSV.\n"; } } + + public function bulkInsert(array $rows) + { + try { + + DB::table('market_data')->insertOrIgnore($rows); + $this->rows = []; + + } catch (\Throwable $e) { + + throw new \Exception('Error: '.$e->getMessage()); + } + } } diff --git a/database/seeders/market_data_seed.csv b/database/seeders/market_data_seed.csv index 174194c..4623360 100644 --- a/database/seeders/market_data_seed.csv +++ b/database/seeders/market_data_seed.csv @@ -241,8 +241,6 @@ AKA,a.k.a. Brands Holding Corp. Common Stock,,2021,Consumer Discretionary,Catalo AKAM,Akamai Technologies Inc. Common Stock,United States,1999,Consumer Discretionary,Business Services AKAN,Akanda Corp. Common Shares,United Kingdom,2022,Health Care, Medicinal Chemicals and Botanical Products AKBA,Akebia Therapeutics Inc. Common Stock,United States,2014,Health Care,Biotechnology: Pharmaceutical Preparations -AKO-A,Embotelladora Andina S.A.,Chile,,, -AKO-B,Embotelladora Andina S.A.,Chile,,, AKR,Acadia Realty Trust Common Stock,United States,,Real Estate,Real Estate Investment Trusts AKRO,Akero Therapeutics Inc. Common Stock,United States,2019,Health Care,Biotechnology: Pharmaceutical Preparations AKTS,Akoustis Technologies Inc. Common Stock,United States,,Utilities,Telecommunications Equipment @@ -789,8 +787,6 @@ BERY,Berry Global Group Inc. Common Stock,United States,2012,Industrials,Plastic BEST,BEST Inc. American Depositary Shares each representing twenty (20) Class A Ordinary Shares,Cayman Islands,2023,Industrials,Trucking Freight/Courier Services BETR,Better Home & Finance Holding Company Class A Common Stock,United Kingdom,2021,Finance,Finance: Consumer Services BETRW,Better Home & Finance Holding Company Warrant,United Kingdom,2021,Finance,Finance: Consumer Services -BF-A,Brown Forman Corporation,United States,,, -BF-B,Brown Forman Corporation,United States,,, BFAC,Battery Future Acquisition Corp. Class A Ordinary Shares,,2022,Finance,Blank Checks BFAM,Bright Horizons Family Solutions Inc. Common Stock,United States,2013,Consumer Discretionary,Other Consumer Services BFC,Bank First Corporation Common Stock,United States,,Finance,Major Banks @@ -1004,8 +1000,6 @@ BREA,Brera Holdings PLC Class B Ordinary Shares,Ireland,2023,Consumer Discretion BRFH,Barfresh Food Group Inc. Common Stock,United States,,Consumer Staples,Packaged Foods BRFS,BRF S.A.,Brazil,,Consumer Staples,Meat/Poultry/Fish BRID,Bridgford Foods Corporation Common Stock,United States,,Consumer Staples,Specialty Foods -BRK-A,Berkshire Hathaway Inc.,United States,,, -BRK-B,Berkshire Hathaway Inc.,United States,,, BRKH,BurTech Acquisition Corp. Class A Common Stock,United States,2022,Finance,Blank Checks BRKHU,BurTech Acquisition Corp. Unit,United States,2021,Finance,Blank Checks BRKHW,BurTech Acquisition Corp. Warrants,United States,2022,Finance,Blank Checks @@ -1542,8 +1536,6 @@ CRBP,Corbus Pharmaceuticals Holdings Inc. Common Stock,United States,,Health Car CRBU,Caribou Biosciences Inc. Common Stock,United States,2021,Health Care,Biotechnology: Biological Products (No Diagnostic Substances) CRC,California Resources Corporation Common Stock,United States,2020,Energy,Oil & Gas Production CRCT,Cricut Inc. Class A Common Stock,United States,2021,Technology,Industrial Machinery/Components -CRD-A,Crawford & Company,United States,,, -CRD-B,Crawford & Company,United States,,, CRDF,Cardiff Oncology Inc. Common Stock,United States,,Health Care,Biotechnology: Biological Products (No Diagnostic Substances) CRDL,Cardiol Therapeutics Inc. Class A Common Shares,Canada,,Health Care,Biotechnology: Biological Products (No Diagnostic Substances) CRDO,Credo Technology Group Holding Ltd Ordinary Shares,United States,2022,Technology,Semiconductors @@ -2850,7 +2842,6 @@ HE,Hawaiian Electric Industries Inc. Common Stock,United States,,Utilities,Elect HEAR,Turtle Beach Corporation Common Stock,United States,,Telecommunications,Telecommunications Equipment HEES,H&E Equipment Services Inc. Common Stock,United States,2006,Industrials,Misc Corporate Leasing Services HEI,Heico Corporation Common Stock,United States,2000,Industrials,Aerospace -HEI-A,Heico Corporation,United States,,, HELE,Helen of Troy Limited Common Stock,Bermuda,,Consumer Discretionary,Home Furnishings HEPA,Hepion Pharmaceuticals Inc. Common Stock,United States,,Health Care,Biotechnology: Pharmaceutical Preparations HEPS,D-Market Electronic Services & Trading American Depositary Shares,Turkey,2021,Consumer Discretionary,Catalog/Specialty Distribution @@ -3011,7 +3002,6 @@ HUnited States,Houston American Energy Corporation Common Stock,United States,,E HUT,Hut 8 Corp. Common Stock,United States,,Finance,Finance: Consumer Services HUYA,HUYA Inc. American depositary shares each representing one Class A ordinary share,United States,2018,Technology,Computer Software: Programming Data Processing HVT,Haverty Furniture Companies Inc. Common Stock,United States,,Consumer Discretionary,Other Specialty Stores -HVT-A,Haverty Furniture Companies Inc.,United States,,, HWBK,Hawthorn Bancshares Inc. Common Stock,United States,,Finance,Major Banks HWC,Hancock Whitney Corporation Common Stock,United States,,Finance,Major Banks HWCPZ,Hancock Whitney Corporation 6.25% Subordinated Notes due 2060,United States,,Finance,Major Banks @@ -11488,7 +11478,6 @@ NYNYR,Empire Resorts Inc.,United States,,, LLEX,"Lilis Energy, Inc.",United States,,Independent Oil & Gas, EZT,"Entergy Texas, Inc.",United States,,, EXPN.L,Experian plc,United Kingdom,,Business Services, -ETX,Eaton Vance Municipal Income 2028 Term Trust,United States,,, EMI,Eaton Vance Michigan Municipal Income Trust,United States,,, EMG,"Emergent Capital, Inc.",United States,,, EMCF,Emclaire Financial Corp.,United States,,Regional - Northeast Banks, @@ -11531,7 +11520,6 @@ ENZY,Enzymotec Ltd.,United States,,Biotechnology, ENBP,ENB Financial Corp,United States,,Regional - Northeast Banks, EMP-A.TO,Empire Company Limited,Canada,,Gold, ECCA,Eagle Point Credit Company Inc.,United States,,, -ECC,Eagle Point Credit Company Inc.,United States,,, NESV,"National Energy Services, Inc.",United States,,, YECO,Yulong Eco-Materials Limited,United States,,, TIK,Tel-Instrument Electronics Corp.,United States,,Aerospace/Defense - Major Diversified, @@ -13136,7 +13124,6 @@ HRG,"HRG Group, Inc.",United States,,Conglomerates, GSS,Golden Star Resources Ltd.,United States,,Gold, GIG,"GigPeak, Inc.",United States,,, SGEN,"Seattle Genetics, Inc.",United States,,Biotechnology, -SAND,Sandstorm Gold Ltd.,United States,,Gold, PGEM,"Ply Gem Holdings, Inc",United States,,General Building Materials, ININ,"Interactive Intelligence Group, Inc.",United States,,, SNR,New Senior Investment Group Inc.,United States,,, @@ -22537,7 +22524,6 @@ WWSG,Worldwide Strategies Incorporated,United States,,Diversified Communication WWTH,"With, Inc.",United States,,, WUMSF,Wumart Stores Inc.,United States,,, WSSH,West Shore Bank Corp.,United States,,, -WSO-B,"Watsco, Inc.",United States,,, WRCKF,WESC AB,United States,,, WMSI,"Williams Industries, Incorporated",United States,,, WLKR,Walker Innovation Inc.,United States,,,