wip test

.github/workflows/build-and-push-images.yml

@@ -43,16 +43,7 @@ jobs:
       - name: Extract version from tag
         id: extract-version
         run: |
-          VERSION="${GITHUB_REF_NAME#v}"
+          echo "version=${GITHUB_REF_NAME#v}" >> $GITHUB_ENV
-      
-          TAGS="investbrainapp/investbrain:${VERSION},ghcr.io/investbrainapp/investbrain:${VERSION}"
-      
-          # Conditionally add 'latest' tags unless 'pre-release' is in the version
-          if [[ "${GITHUB_REF_NAME}" != *alpha* && "${GITHUB_REF_NAME}" != *beta* && "${GITHUB_REF_NAME}" != *rc* ]]; then
-            TAGS="$TAGS,investbrainapp/investbrain:latest,ghcr.io/investbrainapp/investbrain:latest"
-          fi
-      
-          echo "tags=$TAGS" >> $GITHUB_OUTPUT
 
       - name: Build and push
         uses: docker/build-push-action@v6
@@ -60,5 +51,8 @@ jobs:
           platforms: linux/amd64,linux/arm64
           file: ./docker/Dockerfile
           push: true
-          tags: ${{ steps.extract-version.outputs.tags }}
+          tags: |
-
+            investbrainapp/investbrain:latest
+            investbrainapp/investbrain:${{ env.version }}
+            ghcr.io/investbrainapp/investbrain:latest
+            ghcr.io/investbrainapp/investbrain:${{ env.version }}

app/Console/Commands/RefreshMarketData.php

@@ -7,6 +7,7 @@ namespace App\Console\Commands;
 use App\Models\Holding;
 use App\Models\MarketData;
 use Illuminate\Console\Command;
+use Illuminate\Support\Facades\Log;
 
 class RefreshMarketData extends Command
 {
@@ -60,7 +61,7 @@ class RefreshMarketData extends Command
             try {
                 MarketData::getMarketData($holding->symbol, $force);
             } catch (\Throwable $e) {
-                $this->line('Could not refresh '.$holding->symbol.' ('.$e->getMessage().')');
+                Log::error('Could not refresh '.$holding->symbol.' ('.$e->getMessage().')');
             }
         }
     }

app/Exports/Sheets/ConfigSheet.php

@@ -47,13 +47,12 @@ class ConfigSheet implements FromCollection, WithHeadings, WithTitle
         ]);
 
         // reinvested holdings
-        $reinvested_holdings = Holding::myHoldings()->where('reinvest_dividends', true)->get(['portfolio_id', 'symbol']);
+        Holding::myHoldings()->where('reinvest_dividends', true)->get()->each(function ($holding) use (&$configs) {
-        if ($reinvested_holdings->isNotEmpty()) {
             $configs->push([
                 'key' => 'reinvested_dividends',
-                'value' => $reinvested_holdings->toJson(),
+                'value' => $holding->id,
             ]);
-        }
+        });
 
         return $configs;
     }

app/Imports/Sheets/ConfigSheet.php

@@ -54,17 +54,11 @@ class ConfigSheet implements SkipsEmptyRows, ToCollection, WithEvents, WithHeadi
                     $this->backupImport->user->save();
                     break;
 
-                case 'reinvested_dividends':
+                case 'reinvest_dividends':
-                    if (json_validate($config['value'])) {
+
-                        foreach (json_decode($config['value'], true) as $reinvest) {
+                    Holding::myHoldings()->where('id', $config['value'])->update([
-                            Holding::myHoldings($this->backupImport->user->id)
-                                ->where('portfolio_id', $reinvest['portfolio_id'])
-                                ->where('symbol', $reinvest['symbol'])
-                                ->update([
                         'reinvest_dividends' => true,
                     ]);
-                        }
-                    }
                     break;
 
                 default:

app/Imports/Sheets/TransactionsSheet.php

@@ -48,7 +48,7 @@ class TransactionsSheet implements SkipsEmptyRows, ToCollection, WithEvents, Wit
         // if has any transactions not in base currency, need to sync timeseries conversion rates
         if ($transactions->where('currency', '!=', config('investbrain.base_currency'))->isNotEmpty()) {
 
-            CurrencyRate::timeSeriesRates('', $transactions->min('date'), $transactions->max('date'));
+            CurrencyRate::timeSeriesRates('', $transactions->min('date'));
         }
 
         $totalBatches = count($transactions) / $this->batchSize();

app/Models/CurrencyRate.php

@@ -111,7 +111,7 @@ class CurrencyRate extends Model
      *
      * @return array<string, float>
      */
-    public static function timeSeriesRates(string|array|null $currency = null, mixed $start = null, mixed $end = null): array
+    public static function timeSeriesRates(string $currency, mixed $start = null, mixed $end = null): array
     {
         if (empty($start)) {
             return [];
@@ -119,51 +119,47 @@ class CurrencyRate extends Model
 
         $end = $end ?? now();
 
+        dump('Creating period');
+
         $period = CarbonPeriod::create($start, $end);
 
         // No need to send network request - just generate 1s
         if ($currency === config('investbrain.base_currency')) {
 
+            dump('same curr');
+
             $dateRange = [];
             foreach ($period as $date) {
+
                 $dateRange[$date->toDateString()] = 1;
             }
 
             return $dateRange;
         }
 
-        if (is_array($currency)) {
+        dump('diff curr');
-
-            foreach ($currency as $curr) {
-
-                dispatch(fn () => self::timeSeriesRates($curr, $start, $end));
-            }
-
-            return [];
-        }
-
-        // handle currency alias
-        if (! empty($currency)) {
 
         [$currency, $adjustment] = self::getCurrencyAliasAdjustments($currency);
 
-        } else {
+        $currencies = Currency::all()->pluck('currency')->toArray();
 
-            $currency = Currency::all()->pluck('currency')->toArray();
+        dump('got currencies');
-        }
 
-        // get rates
+        // call api in chunks
-        $rates = Frankfurter::setSymbols($currency)->timeSeries($period->first(), $period->last());
+        foreach (collect($period)->chunk(500) as $chunk) {
 
-        $rates = collect(Arr::get($rates, 'rates', []))->sortKeys()->toArray();
+            dump('calling frankf time series');
 
-        $datesOnly = array_keys($rates);
+            $chunkRates = Frankfurter::setSymbols($currencies)->timeSeries($chunk->min(), $chunk->max());
+
+            $rates = Arr::get($chunkRates, 'rates', []);
 
             // loop through each date
             $updates = [];
-        foreach ($period as $date) {
 
-            $lookupDate = self::getNearestPastDate($date, $datesOnly, $rates);
+            foreach ($chunk as $date) {
+
+                $lookupDate = self::getNearestPastDate($date, $rates);
 
                 if (is_null($lookupDate)) {
                     continue;
@@ -181,58 +177,54 @@ class CurrencyRate extends Model
                         'created_at' => now()->toDateTimeString(),
                     ];
                 }
-        }
+
+                dump('inserting');
 
                 // persist
                 self::chunkInsert($updates);
 
-        if (is_string($currency)) {
+            }
+        }
+
+        dump('done');
 
         return collect($updates)
             ->whereBetween('date', [$start, $end ?? now()])
             ->where('currency', $currency)
             ->mapWithKeys(fn ($rate) => [
-                    $rate['date'] => $rate['rate'] * ($adjustment ?? 1),
+                $rate['date'] => $rate['rate'] * $adjustment,
             ])
             ->toArray();
     }
 
-        return [];
+    private static function getNearestPastDate(CarbonInterface $date, array $rates): ?CarbonInterface
-    }
-
-    private static function getNearestPastDate(CarbonInterface $date, array $datesOnly, array $rates): ?CarbonInterface
     {
 
-        // if no dates, nothing to do...
+        $datesWithRates = array_keys($rates);
-        if (empty($datesOnly)) {
+        sort($datesWithRates);
-
-            return null;
-        }
-
-        $mutableDate = $date->copy();
-        $weekAgo = $date->copy()->subWeek();
-        $firstDate = Carbon::parse($datesOnly[0]);
 
         // get rates or find closest valid rate (handles missing weekend rates)
-        while (! isset($rates[$mutableDate->toDateString()])) {
+        while (! isset($rates[$date->toDateString()])) {
-
-            // prevent runaway infinite loops
-            if ($mutableDate->lessThan($weekAgo)) {
-
-                return null;
-            }
 
             // is this the start of a range that falls on a weekend?
-            if ($mutableDate->lessThan($firstDate)) {
+            if ($date->lessThan($first_date = Carbon::parse($datesWithRates[0]))) {
 
-                return $firstDate;
+                $date = $first_date;
+                break;
             }
 
             // try the day before then
-            $mutableDate = $mutableDate->subDay();
+            $date = Carbon::parse($date)->subDay();
+
+            // prevent runaway infinite loops
+            if ($date->lessThan($date->copy()->subWeek())) {
+
+                $date = null;
+                break;
+            }
         }
 
-        return $mutableDate;
+        return $date;
     }
 
     public static function refreshCurrencyData($force = false): void
@@ -273,13 +265,15 @@ class CurrencyRate extends Model
     public static function chunkInsert(array $updates): void
     {
 
-        foreach (array_chunk($updates, 500) as $chunk) {
+        $chunks = array_chunk($updates, 500);
+
+        foreach ($chunks as $chunk) {
 
             QueuedCurrencyRateInsertJob::dispatch($chunk);
         }
     }
 
-    protected static function getCurrencyAliasAdjustments(string $currency)
+    protected static function getCurrencyAliasAdjustments($currency)
     {
         $adjustment = 1;
 

app/Models/Dividend.php

@@ -95,36 +95,49 @@ class Dividend extends Model
             return;
         }
 
+        dump('1. getting div data for '.$symbol);
+        try {
+
             // get some data
             if ($dividend_data = collect() && $start_date && $end_date) {
                 $dividend_data = app(MarketDataInterface::class)->dividends($symbol, $start_date, $end_date);
             }
+        } catch (\Throwable $e) {
+            dump('exception: '.$e->getMessage());
+        }
+
+        dump('2. got div data for '.$symbol);
 
         // ah, we found some dividends...
         if ($dividend_data->isNotEmpty()) {
 
+            $dividend_data = $dividend_data->sortBy('date');
+
+            dump('3. getting mkt data for '.$symbol);
+
             $market_data = MarketData::getMarketData($symbol);
 
-            $dividend_data
+            dump('4. got market data for '.$symbol);
-                ->chunk(10)
+
-                ->each(function ($chunk) use ($market_data) {
+            // todo: use this for start_date - $dividend_data->first()->get('date')
 
             // get historic conversion rates
-                    $rate_to_base = CurrencyRate::timeSeriesRates($market_data->currency, $chunk->min('date'), $chunk->max('date'));
+            $rate_to_base = CurrencyRate::timeSeriesRates($market_data->currency, $dividend_data->first()->get('date'), $end_date);
 
+            dump('5. got time series for '.$symbol);
             // create mass insert
-                    foreach ($chunk as $index => $dividend) {
+            foreach ($dividend_data as $index => $dividend) {
                 $rate_to_base_date = 1 / Arr::get($rate_to_base, Carbon::parse(Arr::get($dividend, 'date'))->toDateString(), 1);
 
                 $dividend['dividend_amount_base'] = $dividend['dividend_amount'] * $rate_to_base_date;
 
-                        $chunk[$index] = [...$dividend, ...['id' => Str::uuid()->toString(), 'updated_at' => now(), 'created_at' => now()]];
+                $dividend_data[$index] = [...$dividend, ...['id' => Str::uuid()->toString(), 'updated_at' => now(), 'created_at' => now()]];
             }
 
             // insert records
-                    (new self)->insertOrIgnore($chunk->toArray());
+            (new self)->insertOrIgnore($dividend_data->toArray());
-                });
 
+            dump('6. inserted  for '.$symbol);
             // sync to holdings
             self::syncHoldings($symbol);
 
@@ -134,7 +147,9 @@ class Dividend extends Model
             // sync last dividend amount to market data table
             $market_data->last_dividend_amount = $dividend_data->sortByDesc('date')->first()['dividend_amount'];
             $market_data->save();
+
         }
+
     }
 
     public static function syncHoldings(string $symbol): void
@@ -191,7 +206,6 @@ class Dividend extends Model
                         'date' => $dividend['date'],
                         'portfolio_id' => $holding->portfolio_id,
                         'symbol' => $holding->symbol,
-                        'currency' => $holding->market_data->currency,
                         'transaction_type' => 'BUY',
                         'reinvested_dividend' => true,
                         'cost_basis' => 0,

app/Models/Portfolio.php

@@ -152,12 +152,7 @@ class Portfolio extends Model
 
         $total_performance = [];
 
-        // get unique currencies for holdings
+        $holdings->each(function ($holding) use (&$total_performance) {
-        foreach ($holdings->groupBy('market_data.currency')->keys() as $currency) {
-            $currency_rates[$currency] = CurrencyRate::timeSeriesRates($currency, $holdings->min('first_transaction_date'), now());
-        }
-
-        $holdings->each(function ($holding) use (&$total_performance, $currency_rates) {
 
             $period = CarbonPeriod::create(
                 $holding->first_transaction_date,
@@ -168,6 +163,7 @@ class Portfolio extends Model
 
             $daily_performance = $holding->dailyPerformance($holding->first_transaction_date, now());
             $all_history = app(MarketDataInterface::class)->history($holding->symbol, $holding->first_transaction_date, now());
+            $currency_rates = CurrencyRate::timeSeriesRates($holding->market_data->currency, $holding->first_transaction_date, now());
 
             $holding_performance = [];
 
@@ -183,7 +179,7 @@ class Portfolio extends Model
                     $holding_performance[$date] = [
                         'date' => $date,
                         'portfolio_id' => $this->id,
-                        'total_market_value' => $total_market_value * (1 / Arr::get($currency_rates[$holding->market_data->currency], $date, 1)),
+                        'total_market_value' => $total_market_value * (1 / Arr::get($currency_rates, $date, 1)),
                     ];
                 }
             }

database/migrations/2025_12_28_000001_add_multi_currency_support.php

@@ -3,7 +3,6 @@
 declare(strict_types=1);
 
 use App\Models\CurrencyRate;
-use App\Models\Holding;
 use App\Models\Transaction;
 use Database\Seeders\CurrencySeeder;
 use Database\Seeders\MarketDataSeeder;
@@ -97,17 +96,17 @@ return new class extends Migration
                 '--force' => true,
             ]);
 
-            Artisan::call('db:seed', [
-                '--class' => MarketDataSeeder::class,
-                '--force' => true,
-            ]);
-
             CurrencyRate::timeSeriesRates(
-                Holding::all()->groupBy('market_data.currency')->keys()->toArray(),
+                '', // use fake currency to force
                 Transaction::min('date')
             );
 
             CurrencyRate::refreshCurrencyData();
+
+            Artisan::call('db:seed', [
+                '--class' => MarketDataSeeder::class,
+                '--force' => true,
+            ]);
         }
 
         /**

database/seeders/MarketDataSeeder.php

@@ -12,6 +12,8 @@ class MarketDataSeeder extends Seeder
 {
     use WithoutModelEvents;
 
+    public array $rows = [];
+
     /**
      * Run the database seeds.
      */
@@ -42,7 +44,7 @@ class MarketDataSeeder extends Seeder
                     $meta_data = json_decode(base64_decode($data['meta_data']), true);
                     $meta_data['source'] = 'market_data_seeder';
 
-                    $rows[] = [
+                    $this->rows[] = [
                         'symbol' => $data['symbol'],
                         'name' => $data['name'],
                         'currency' => $data['currency'],
@@ -52,17 +54,16 @@ class MarketDataSeeder extends Seeder
                     $rowCount++;
 
                     if ($rowCount % $chunkSize == 0) {
-                        $this->bulkInsert($rows);
+                        DB::table('market_data')->upsert($this->rows, ['symbol'], ['name', 'currency', 'meta_data']);
-                        $rows = [];
+                        $this->rows = [];
                     }
                 }
             }
 
             // final clean up
-            if (! empty($rows)) {
+            if (! empty($this->rows)) {
 
-                $this->bulkInsert($rows);
+                $this->bulkInsert($this->rows);
-                $rows = [];
             }
 
             // Close the CSV file
@@ -76,17 +77,16 @@ class MarketDataSeeder extends Seeder
         }
     }
 
-    private function bulkInsert($rows): void
+    public function bulkInsert(array $rows)
     {
         try {
 
-            DB::table('market_data')->upsert($rows, ['symbol'], ['name', 'currency', 'meta_data']);
+            DB::table('market_data')->insertOrIgnore($rows);
+            $this->rows = [];
 
         } catch (\Throwable $e) {
 
             throw new \Exception('Error: '.$e->getMessage());
         }
-
-        gc_collect_cycles();
     }
 }

tests/ImportExportTest.php

@@ -84,6 +84,7 @@ class ImportExportTest extends TestCase
         ]);
 
         $holding = Holding::create([
+            'id' => '9cf8a662-7347-49fb-b9de-0cc1430a8d1f',
             'portfolio_id' => '9e792bb8-94e7-4ed3-b8cc-43b50d34c337',
             'symbol' => 'ACME',
             'quantity' => 0,

tests/MarketDataTest.php

@@ -21,7 +21,7 @@ class MarketDataTest extends TestCase
             '--force' => true,
         ]);
 
-        $this->assertEquals(13187, MarketData::count('symbol'));
+        $this->assertEquals(14464, MarketData::count('symbol'));
     }
 
     public function test_can_get_quote_from_provider()

tests/MultiCurrencyTest.php

@@ -133,8 +133,11 @@ class MultiCurrencyTest extends TestCase
         $portfolio = Portfolio::factory()->create();
         $transaction = Transaction::factory()->buy()->lastYear()->portfolio($portfolio->id)->symbol('ACME')->create();
 
+        $expected_num_calls = count(collect(CarbonPeriod::create($transaction->date, now()))->chunk(500));
+
         Frankfurter::expects('setSymbols')
-            ->andReturnSelf();
+            ->andReturnSelf()
+            ->times($expected_num_calls);
         Frankfurter::expects('timeSeries')
             ->andReturn(['rates' => [
                 now()->subDays(3)->toDateString() => [
@@ -149,7 +152,8 @@ class MultiCurrencyTest extends TestCase
                 now()->toDateString() => [
                     'ZZZ' => .01,
                 ],
-            ]]);
+            ]])
+            ->times($expected_num_calls);
 
         CurrencyRate::timeSeriesRates(
             '', // use fake currency to force
@@ -225,71 +229,8 @@ class MultiCurrencyTest extends TestCase
             ->andReturn(['rates' => $results]);
 
         $result = CurrencyRate::timeSeriesRates('ZZZ', $start, $end);
+
         $this->assertEquals(count($period) - 1, count($result));
-
-        $result = CurrencyRate::all();
-        $this->assertEquals(count($period), count($result));
-    }
-
-    public function test_can_get_time_series_rates_with_null_currency()
-    {
-
-        $start = now()->subWeeks(2);
-        $end = now();
-
-        $period = CarbonPeriod::create($start, $end);
-
-        // mock response from Frankfurter
-        $results = [];
-        collect($period->copy()->filter('isWeekday'))->each(function ($date) use (&$results) {
-            $date = $date->toDateString();
-
-            $results[$date] = [
-                'FOO' => random_int(10, 150) / 1000,
-            ];
-        });
-
-        Frankfurter::expects('setSymbols')
-            ->andReturnSelf();
-        Frankfurter::expects('timeSeries')
-            ->andReturn(['rates' => $results]);
-
-        $result = CurrencyRate::timeSeriesRates(null, $start, $end);
-        $this->assertEquals(0, count($result));
-
-        $result = CurrencyRate::all();
-        $this->assertEquals(count($period), count($result));
-    }
-
-    public function test_can_get_time_series_rates_with_currencies()
-    {
-
-        $start = now()->subWeeks(2);
-        $end = now();
-
-        $period = CarbonPeriod::create($start, $end);
-
-        // mock response from Frankfurter
-        $results = [];
-        collect($period->copy()->filter('isWeekday'))->each(function ($date) use (&$results) {
-            $date = $date->toDateString();
-
-            $results[$date] = [
-                'FOO' => random_int(10, 150) / 1000,
-                'BAR' => random_int(10, 150) / 1000,
-            ];
-        });
-
-        Frankfurter::expects('setSymbols')
-            ->andReturnSelf();
-        Frankfurter::expects('timeSeries')
-            ->andReturn(['rates' => $results]);
-
-        $result = CurrencyRate::timeSeriesRates(null, $start, $end);
-        $this->assertEquals(0, count($result));
-
-        $result = CurrencyRate::all();
-        $this->assertEquals(count($period) * 2, count($result));
     }
 
     public function test_time_series_rate_calls_are_chunked()
@@ -311,9 +252,11 @@ class MultiCurrencyTest extends TestCase
         });
 
         Frankfurter::expects('setSymbols')
-            ->andReturnSelf();
+            ->andReturnSelf()
+            ->times(4);
         Frankfurter::expects('timeSeries')
-            ->andReturn(['rates' => $results]);
+            ->andReturn(['rates' => $results])
+            ->times(4);
 
         CurrencyRate::timeSeriesRates('ZZZ', $start, $end);
     }