javier/investbrain
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: improve seeder (and remove symbol dupes)

Browse Source
This commit is contained in:
hackerESQ
2025-03-07 18:43:55 -06:00
parent 730903c383
commit 00fbdec6f1
2 changed files with 33 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
database/seeders/MarketDataSeeder.php
+33 -22
View File
@@ -12,6 +12,8 @@ class MarketDataSeeder extends Seeder
{
use WithoutModelEvents;
public array $rows = [];
/**
* Run the database seeds.
*/
@@ -26,7 +28,6 @@ class MarketDataSeeder extends Seeder
if (($handle = fopen($csvFilePath, 'r')) !== false) {
$header = null;
$rows = [];
$rowCount = 0;
while (($row = fgetcsv($handle, 0, ',')) !== false) {
@@ -38,36 +39,33 @@ class MarketDataSeeder extends Seeder
} else {
try {
$data = array_combine($header, $row);
$data = array_combine($header, $row);
$rows[] = [
'symbol' => $data['symbol'],
'name' => $data['name'],
'meta_data' => json_encode([
'country' => $data['country'],
'first_trade_year' => $data['first_trade_year'],
'sector' => $data['sector'],
'industry' => $data['industry'],
]),
];
$this->rows[] = [
'symbol' => $data['symbol'],
'name' => $data['name'],
'meta_data' => json_encode([
'country' => $data['country'],
'first_trade_year' => $data['first_trade_year'],
'sector' => $data['sector'],
'industry' => $data['industry'],
]),
];
$rowCount++;
$rowCount++;
if ($rowCount % $chunkSize == 0) {
DB::table('market_data')->insertOrIgnore($rows);
$rows = [];
}
} catch (\Throwable $e) {
if ($rowCount % $chunkSize == 0) {
throw new \Exception('Error: '.$e->getMessage());
$this->bulkInsert($this->rows);
}
}
}
// final clean up
if (! empty($rows)) {
DB::table('market_data')->insertOrIgnore($rows);
if (! empty($this->rows)) {
echo 'did '.$rowCount.' already. doing '.count($this->rows).' more';
$this->bulkInsert($this->rows);
}
// Close the CSV file
@@ -80,4 +78,17 @@ class MarketDataSeeder extends Seeder
echo "Failed to open the CSV.\n";
}
}
public function bulkInsert(array $rows)
{
try {
DB::table('market_data')->insertOrIgnore($rows);
$this->rows = [];
} catch (\Throwable $e) {
throw new \Exception('Error: '.$e->getMessage());
}
}
}