feat:improved background importing

2024-10-24 14:48:24 -05:00
parent 7543c0a865
commit 3c368310ad
15 changed files with 459 additions and 139 deletions
@@ -3,57 +3,69 @@
 namespace App\Imports\Sheets;

 use App\Models\DailyChange;
+use App\Models\BackupImport;
 use Illuminate\Support\Carbon;
-use Illuminate\Support\Collection;
-use Maatwebsite\Excel\Concerns\ToCollection;
-use App\Imports\ValidatesPortfolioPermissions;
+use Illuminate\Support\Facades\DB;
+use Maatwebsite\Excel\Concerns\ToModel;
+use Maatwebsite\Excel\Events\BeforeSheet;
+use Maatwebsite\Excel\Concerns\WithEvents;
+use Maatwebsite\Excel\Concerns\WithUpserts;
+use App\Rules\PortfolioAccessValidationRule;
 use Maatwebsite\Excel\Concerns\SkipsEmptyRows;
 use Maatwebsite\Excel\Concerns\WithHeadingRow;
 use Maatwebsite\Excel\Concerns\WithValidation;
-use Maatwebsite\Excel\Concerns\WithChunkReading;
+use Maatwebsite\Excel\Concerns\WithBatchInserts;

-class DailyChangesSheet implements ToCollection, WithHeadingRow, WithValidation, SkipsEmptyRows, WithChunkReading
+class DailyChangesSheet implements ToModel, WithHeadingRow, WithValidation, WithBatchInserts, WithUpserts, SkipsEmptyRows, WithEvents
 {
-    use ValidatesPortfolioPermissions;
+    public function __construct(
+        public BackupImport $backupImport
+    ) { }

-    public function collection(Collection $dailyChanges)
+    /**
+     * @return array
+     */
+    public function registerEvents(): array
    {
-        $this->validatePortfolioPermissions($dailyChanges);
+        return [
+            BeforeSheet::class => function(BeforeSheet $event) {
+                DB::commit();
+                $this->backupImport->update([
+                    'message' => __('Importing daily changes...'),
+                ]);
+                DB::beginTransaction();
+            }
+        ];
+    }
+    
+    public function model(array $dailyChange)
+    {
+        return new DailyChange([
+            'total_market_value' => $dailyChange['total_market_value'],
+            'total_cost_basis' => $dailyChange['total_cost_basis'],
+            'total_gain' => $dailyChange['total_gain'],
+            'total_dividends_earned' => $dailyChange['total_dividends_earned'],
+            'realized_gains' => $dailyChange['realized_gains'],
+            'annotation' => $dailyChange['annotation'],
+            'portfolio_id' => $dailyChange['portfolio_id'],
+            'date' => Carbon::parse($dailyChange['date'])->format('Y-m-d')
+        ]);
+    }
+    
+    public function batchSize(): int
+    {
+        return 150;
+    }

-        $chunkSize = 500;
-
-        $dailyChanges->chunk($chunkSize)->each(function ($chunk) {
-
-            // have to manually format dates since we're doing a raw upsert
-            $chunk = $chunk->map(function ($daily) {
-
-                $daily['date'] = Carbon::parse($daily['date'])->format('Y-m-d');
-
-                return $daily;
-            });
-
-            DailyChange::upsert(
-                $chunk->toArray(),
-                [
-                    'portfolio_id',
-                    'date'
-                ],
-                [
-                    'total_market_value',
-                    'total_cost_basis',
-                    'total_gain',
-                    'total_dividends_earned',
-                    'realized_gains',
-                    'annotation'
-                ]
-            );
-        });
+    public function uniqueBy()
+    {
+        return ['portfolio_id', 'date'];
    }

    public function rules(): array
    {
        return [
-            'portfolio_id' => ['required', 'exists:portfolios,id'],
+            'portfolio_id' => ['required', new PortfolioAccessValidationRule($this->backupImport->user_id)], 
            'date' => ['required', 'date'],
            'total_market_value' => ['sometimes', 'nullable', 'numeric'],
            'total_cost_basis' => ['sometimes', 'nullable', 'min:0', 'numeric'],
@@ -63,9 +75,4 @@ class DailyChangesSheet implements ToCollection, WithHeadingRow, WithValidation,
            'annotation' => ['sometimes', 'nullable', 'string'],
        ];
    }
-
-    public function chunkSize(): int
-    {
-        return 500;
-    }
 }
@@ -2,24 +2,48 @@

 namespace App\Imports\Sheets;

-use App\Console\Commands\SyncDailyChange;
 use App\Models\Portfolio;
+use App\Models\BackupImport;
 use Illuminate\Support\Collection;
+use Illuminate\Support\Facades\DB;
 use Illuminate\Support\Facades\Artisan;
+use App\Console\Commands\SyncDailyChange;
+use Maatwebsite\Excel\Events\BeforeSheet;
+use Maatwebsite\Excel\Concerns\WithEvents;
 use Maatwebsite\Excel\Concerns\ToCollection;
 use Maatwebsite\Excel\Concerns\SkipsEmptyRows;
 use Maatwebsite\Excel\Concerns\WithHeadingRow;
 use Maatwebsite\Excel\Concerns\WithValidation;

-class PortfoliosSheet implements ToCollection, WithValidation, WithHeadingRow, SkipsEmptyRows
+class PortfoliosSheet implements ToCollection, WithValidation, WithHeadingRow, SkipsEmptyRows, WithEvents
 {
+    public function __construct(
+        public BackupImport $backupImport
+    ) { }
+    
+    /**
+     * @return array
+     */
+    public function registerEvents(): array
+    {
+        return [
+            BeforeSheet::class => function(BeforeSheet $event) {
+                DB::commit();
+                $this->backupImport->update([
+                    'message' => __('Importing portfolios...'),
+                ]);
+                DB::beginTransaction();
+            }
+        ];
+    }
+
    public function collection(Collection $portfolios)
    {
        foreach ($portfolios as $index => $portfolio) {
-            
+
            Portfolio::unguard();

-            $portfolio = Portfolio::updateOrCreate([
+            $portfolio = Portfolio::fullAccess($this->backupImport->user_id)->updateOrCreate([
                'id' => $portfolio['portfolio_id']
            ], [
                'id' => $portfolio['portfolio_id'] ?? null,
@@ -2,51 +2,80 @@

 namespace App\Imports\Sheets;

+use App\Models\Holding;
 use App\Models\Transaction;
-use Illuminate\Support\Collection;
-use Maatwebsite\Excel\Concerns\ToCollection;
-use App\Imports\ValidatesPortfolioPermissions;
+use Illuminate\Support\Str;
+use App\Models\BackupImport;
+use Illuminate\Support\Carbon;
+use Illuminate\Support\Facades\DB;
+use Maatwebsite\Excel\Concerns\ToModel;
+use Maatwebsite\Excel\Events\BeforeSheet;
+use Maatwebsite\Excel\Concerns\WithUpserts;
+use App\Rules\PortfolioAccessValidationRule;
 use Maatwebsite\Excel\Concerns\SkipsEmptyRows;
 use Maatwebsite\Excel\Concerns\WithHeadingRow;
 use Maatwebsite\Excel\Concerns\WithValidation;
-use Maatwebsite\Excel\Concerns\WithChunkReading;
+use Maatwebsite\Excel\Concerns\WithBatchInserts;
+use Maatwebsite\Excel\Concerns\WithEvents;

-class TransactionsSheet implements ToCollection, WithHeadingRow, WithValidation, SkipsEmptyRows, WithChunkReading
+class TransactionsSheet implements ToModel, WithHeadingRow, WithValidation, WithBatchInserts, WithUpserts, SkipsEmptyRows, WithEvents
 {
-    use ValidatesPortfolioPermissions;

-    public function collection(Collection $transactions)
+    public function __construct(
+        public BackupImport $backupImport
+    ) { }
+
+    /**
+     * @return array
+     */
+    public function registerEvents(): array
    {
-        $this->validatePortfolioPermissions($transactions);
-        
-        Transaction::withoutEvents(function () use ($transactions) {
-
-            foreach ($transactions->sortBy('date') as $transaction) {
-
-                Transaction::where('id', $transaction['transaction_id'])
-                        ->firstOr(function () use ($transaction) {
-
-                            $transaction = Transaction::make()->forceFill([
-                                'id' => $transaction['transaction_id'],
-                                'symbol' => $transaction['symbol'],
-                                'portfolio_id' => $transaction['portfolio_id'],
-                                'transaction_type' => $transaction['transaction_type'],
-                                'quantity' => $transaction['quantity'],
-                                'cost_basis' => $transaction['cost_basis'] ?? 0,
-                                'sale_price' => $transaction['sale_price'],
-                                'split' => $transaction['split'] ?? null,
-                                'reinvested_dividend' => $transaction['reinvested_dividend'] ?? null,
-                                'date' => $transaction['date'],
-                            ]);
-
-                            $transaction->save();
-
-                            return $transaction;
-                        })
-                        ->syncToHolding();
+        return [
+            BeforeSheet::class => function(BeforeSheet $event) {
+                DB::commit();
+                $this->backupImport->update([
+                    'message' => __('Importing transactions...'),
+                ]);
+                DB::beginTransaction();
            }
-        });
-        
+        ];
+    }
+
+    public function model(array $transaction)
+    {
+        $transaction = new Transaction([
+            'id' => $transaction['transaction_id'] ?? Str::uuid()->toString(),
+            'symbol' => strtoupper($transaction['symbol']),
+            'portfolio_id' => $transaction['portfolio_id'],
+            'transaction_type' => $transaction['transaction_type'],
+            'quantity' => $transaction['quantity'],
+            'cost_basis' => $transaction['cost_basis'] ?? 0,
+            'sale_price' => $transaction['sale_price'],
+            'split' => boolval($transaction['split']) ? 1 : 0,
+            'reinvested_dividend' => boolval($transaction['reinvested_dividend']) ? 1 : 0,
+            'date' => Carbon::parse($transaction['date'])->format('Y-m-d')
+        ]);
+
+        // stub out related holding
+        Holding::firstOrCreate([
+            'symbol' => $transaction->symbol,
+            'portfolio_id' => $transaction->portfolio_id
+        ], [
+            'quantity' => 0,
+            'average_cost_basis' => 0,
+        ]);
+
+        return $transaction;
+    }
+    
+    public function batchSize(): int
+    {
+        return 150;
+    }
+
+    public function uniqueBy()
+    {
+        return 'id';
    }

    public function rules(): array
@@ -54,7 +83,7 @@ class TransactionsSheet implements ToCollection, WithHeadingRow, WithValidation,
        return [
            'transaction_id' => ['sometimes', 'nullable'],
            'symbol' => ['required', 'string'],
-            'portfolio_id' => ['required', 'exists:portfolios,id'],
+            'portfolio_id' => ['required', new PortfolioAccessValidationRule($this->backupImport->user_id)],
            'quantity' => ['required', 'min:0', 'numeric'],
            'transaction_type' => ['required', 'in:BUY,SELL'],
            'date' => ['required', 'date'],
@@ -65,9 +94,4 @@ class TransactionsSheet implements ToCollection, WithHeadingRow, WithValidation,
            'sale_price' => ['sometimes', 'nullable', 'min:0', 'numeric'],
        ];
    }
-
-    public function chunkSize(): int
-    {
-        return 500;
-    }
 }