diff --git a/app/Imports/BackupImport.php b/app/Imports/BackupImport.php index 686a3d6..9dca01b 100644 --- a/app/Imports/BackupImport.php +++ b/app/Imports/BackupImport.php @@ -4,13 +4,17 @@ namespace App\Imports; use App\Imports\Sheets\PortfoliosSheet; use Illuminate\Support\Facades\Artisan; +use App\Console\Commands\SyncHoldingData; use App\Imports\Sheets\DailyChangesSheet; use App\Imports\Sheets\TransactionsSheet; use Maatwebsite\Excel\Events\AfterImport; use Maatwebsite\Excel\Concerns\Importable; use Maatwebsite\Excel\Concerns\WithEvents; +use Maatwebsite\Excel\Events\BeforeImport; +use Maatwebsite\Excel\Events\ImportFailed; use App\Console\Commands\RefreshMarketData; use App\Console\Commands\RefreshDividendData; +use App\Models\BackupImport as BackupImportModel; use Maatwebsite\Excel\Concerns\WithMultipleSheets; class BackupImport implements WithMultipleSheets, WithEvents @@ -18,25 +22,51 @@ class BackupImport implements WithMultipleSheets, WithEvents use Importable; + public function __construct( + public BackupImportModel $backupImportModel + ) { } + /** * @return array */ public function registerEvents(): array { return [ - AfterImport::class => - fn() => Artisan::queue(RefreshMarketData::class, ['--force' => true])->chain([ - fn() => Artisan::call(RefreshDividendData::class, ['--force' => true]) - ]) + BeforeImport::class => fn() => $this->backupImportModel->update([ + 'status' => 'in_progress', + 'message' => __('Import is in progress...'), + ]), + AfterImport::class => function () { + + $this->backupImportModel->update([ + 'status' => 'success', + 'message' => 'Import completed successfully!', + 'completed_at' => now() + ]); + + Artisan::queue(RefreshMarketData::class, ['--user' => $this->backupImportModel->user_id, '--force' => true]) + ->chain([ + fn() => Artisan::call(RefreshDividendData::class, ['--user' => $this->backupImportModel->user_id, '--force' => true]) + ]) + ->chain([ + fn() => Artisan::call(SyncHoldingData::class, ['--user' => $this->backupImportModel->user_id]) + ]); + }, + ImportFailed::class => fn(ImportFailed $event) => $this->backupImportModel->update([ + 'status' => 'failed', + 'message' => 'Error: '. substr($event->getException()->getMessage(), 0, 220), + 'has_errors' => true, + 'completed_at' => now() + ]), ]; } public function sheets(): array { return [ - 'Portfolios' => new PortfoliosSheet, - 'Transactions' => new TransactionsSheet, - 'Daily Changes' => new DailyChangesSheet, + 'Portfolios' => new PortfoliosSheet($this->backupImportModel), + 'Transactions' => new TransactionsSheet($this->backupImportModel), + 'Daily Changes' => new DailyChangesSheet($this->backupImportModel), ]; } } diff --git a/app/Imports/Sheets/DailyChangesSheet.php b/app/Imports/Sheets/DailyChangesSheet.php index 5cc7961..51fbde7 100644 --- a/app/Imports/Sheets/DailyChangesSheet.php +++ b/app/Imports/Sheets/DailyChangesSheet.php @@ -3,57 +3,69 @@ namespace App\Imports\Sheets; use App\Models\DailyChange; +use App\Models\BackupImport; use Illuminate\Support\Carbon; -use Illuminate\Support\Collection; -use Maatwebsite\Excel\Concerns\ToCollection; -use App\Imports\ValidatesPortfolioPermissions; +use Illuminate\Support\Facades\DB; +use Maatwebsite\Excel\Concerns\ToModel; +use Maatwebsite\Excel\Events\BeforeSheet; +use Maatwebsite\Excel\Concerns\WithEvents; +use Maatwebsite\Excel\Concerns\WithUpserts; +use App\Rules\PortfolioAccessValidationRule; use Maatwebsite\Excel\Concerns\SkipsEmptyRows; use Maatwebsite\Excel\Concerns\WithHeadingRow; use Maatwebsite\Excel\Concerns\WithValidation; -use Maatwebsite\Excel\Concerns\WithChunkReading; +use Maatwebsite\Excel\Concerns\WithBatchInserts; -class DailyChangesSheet implements ToCollection, WithHeadingRow, WithValidation, SkipsEmptyRows, WithChunkReading +class DailyChangesSheet implements ToModel, WithHeadingRow, WithValidation, WithBatchInserts, WithUpserts, SkipsEmptyRows, WithEvents { - use ValidatesPortfolioPermissions; + public function __construct( + public BackupImport $backupImport + ) { } - public function collection(Collection $dailyChanges) + /** + * @return array + */ + public function registerEvents(): array { - $this->validatePortfolioPermissions($dailyChanges); + return [ + BeforeSheet::class => function(BeforeSheet $event) { + DB::commit(); + $this->backupImport->update([ + 'message' => __('Importing daily changes...'), + ]); + DB::beginTransaction(); + } + ]; + } + + public function model(array $dailyChange) + { + return new DailyChange([ + 'total_market_value' => $dailyChange['total_market_value'], + 'total_cost_basis' => $dailyChange['total_cost_basis'], + 'total_gain' => $dailyChange['total_gain'], + 'total_dividends_earned' => $dailyChange['total_dividends_earned'], + 'realized_gains' => $dailyChange['realized_gains'], + 'annotation' => $dailyChange['annotation'], + 'portfolio_id' => $dailyChange['portfolio_id'], + 'date' => Carbon::parse($dailyChange['date'])->format('Y-m-d') + ]); + } + + public function batchSize(): int + { + return 150; + } - $chunkSize = 500; - - $dailyChanges->chunk($chunkSize)->each(function ($chunk) { - - // have to manually format dates since we're doing a raw upsert - $chunk = $chunk->map(function ($daily) { - - $daily['date'] = Carbon::parse($daily['date'])->format('Y-m-d'); - - return $daily; - }); - - DailyChange::upsert( - $chunk->toArray(), - [ - 'portfolio_id', - 'date' - ], - [ - 'total_market_value', - 'total_cost_basis', - 'total_gain', - 'total_dividends_earned', - 'realized_gains', - 'annotation' - ] - ); - }); + public function uniqueBy() + { + return ['portfolio_id', 'date']; } public function rules(): array { return [ - 'portfolio_id' => ['required', 'exists:portfolios,id'], + 'portfolio_id' => ['required', new PortfolioAccessValidationRule($this->backupImport->user_id)], 'date' => ['required', 'date'], 'total_market_value' => ['sometimes', 'nullable', 'numeric'], 'total_cost_basis' => ['sometimes', 'nullable', 'min:0', 'numeric'], @@ -63,9 +75,4 @@ class DailyChangesSheet implements ToCollection, WithHeadingRow, WithValidation, 'annotation' => ['sometimes', 'nullable', 'string'], ]; } - - public function chunkSize(): int - { - return 500; - } } diff --git a/app/Imports/Sheets/PortfoliosSheet.php b/app/Imports/Sheets/PortfoliosSheet.php index e98801d..53aa030 100644 --- a/app/Imports/Sheets/PortfoliosSheet.php +++ b/app/Imports/Sheets/PortfoliosSheet.php @@ -2,24 +2,48 @@ namespace App\Imports\Sheets; -use App\Console\Commands\SyncDailyChange; use App\Models\Portfolio; +use App\Models\BackupImport; use Illuminate\Support\Collection; +use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Artisan; +use App\Console\Commands\SyncDailyChange; +use Maatwebsite\Excel\Events\BeforeSheet; +use Maatwebsite\Excel\Concerns\WithEvents; use Maatwebsite\Excel\Concerns\ToCollection; use Maatwebsite\Excel\Concerns\SkipsEmptyRows; use Maatwebsite\Excel\Concerns\WithHeadingRow; use Maatwebsite\Excel\Concerns\WithValidation; -class PortfoliosSheet implements ToCollection, WithValidation, WithHeadingRow, SkipsEmptyRows +class PortfoliosSheet implements ToCollection, WithValidation, WithHeadingRow, SkipsEmptyRows, WithEvents { + public function __construct( + public BackupImport $backupImport + ) { } + + /** + * @return array + */ + public function registerEvents(): array + { + return [ + BeforeSheet::class => function(BeforeSheet $event) { + DB::commit(); + $this->backupImport->update([ + 'message' => __('Importing portfolios...'), + ]); + DB::beginTransaction(); + } + ]; + } + public function collection(Collection $portfolios) { foreach ($portfolios as $index => $portfolio) { - + Portfolio::unguard(); - $portfolio = Portfolio::updateOrCreate([ + $portfolio = Portfolio::fullAccess($this->backupImport->user_id)->updateOrCreate([ 'id' => $portfolio['portfolio_id'] ], [ 'id' => $portfolio['portfolio_id'] ?? null, diff --git a/app/Imports/Sheets/TransactionsSheet.php b/app/Imports/Sheets/TransactionsSheet.php index 09a23bc..9e6ace0 100644 --- a/app/Imports/Sheets/TransactionsSheet.php +++ b/app/Imports/Sheets/TransactionsSheet.php @@ -2,51 +2,80 @@ namespace App\Imports\Sheets; +use App\Models\Holding; use App\Models\Transaction; -use Illuminate\Support\Collection; -use Maatwebsite\Excel\Concerns\ToCollection; -use App\Imports\ValidatesPortfolioPermissions; +use Illuminate\Support\Str; +use App\Models\BackupImport; +use Illuminate\Support\Carbon; +use Illuminate\Support\Facades\DB; +use Maatwebsite\Excel\Concerns\ToModel; +use Maatwebsite\Excel\Events\BeforeSheet; +use Maatwebsite\Excel\Concerns\WithUpserts; +use App\Rules\PortfolioAccessValidationRule; use Maatwebsite\Excel\Concerns\SkipsEmptyRows; use Maatwebsite\Excel\Concerns\WithHeadingRow; use Maatwebsite\Excel\Concerns\WithValidation; -use Maatwebsite\Excel\Concerns\WithChunkReading; +use Maatwebsite\Excel\Concerns\WithBatchInserts; +use Maatwebsite\Excel\Concerns\WithEvents; -class TransactionsSheet implements ToCollection, WithHeadingRow, WithValidation, SkipsEmptyRows, WithChunkReading +class TransactionsSheet implements ToModel, WithHeadingRow, WithValidation, WithBatchInserts, WithUpserts, SkipsEmptyRows, WithEvents { - use ValidatesPortfolioPermissions; - public function collection(Collection $transactions) + public function __construct( + public BackupImport $backupImport + ) { } + + /** + * @return array + */ + public function registerEvents(): array { - $this->validatePortfolioPermissions($transactions); - - Transaction::withoutEvents(function () use ($transactions) { - - foreach ($transactions->sortBy('date') as $transaction) { - - Transaction::where('id', $transaction['transaction_id']) - ->firstOr(function () use ($transaction) { - - $transaction = Transaction::make()->forceFill([ - 'id' => $transaction['transaction_id'], - 'symbol' => $transaction['symbol'], - 'portfolio_id' => $transaction['portfolio_id'], - 'transaction_type' => $transaction['transaction_type'], - 'quantity' => $transaction['quantity'], - 'cost_basis' => $transaction['cost_basis'] ?? 0, - 'sale_price' => $transaction['sale_price'], - 'split' => $transaction['split'] ?? null, - 'reinvested_dividend' => $transaction['reinvested_dividend'] ?? null, - 'date' => $transaction['date'], - ]); - - $transaction->save(); - - return $transaction; - }) - ->syncToHolding(); + return [ + BeforeSheet::class => function(BeforeSheet $event) { + DB::commit(); + $this->backupImport->update([ + 'message' => __('Importing transactions...'), + ]); + DB::beginTransaction(); } - }); - + ]; + } + + public function model(array $transaction) + { + $transaction = new Transaction([ + 'id' => $transaction['transaction_id'] ?? Str::uuid()->toString(), + 'symbol' => strtoupper($transaction['symbol']), + 'portfolio_id' => $transaction['portfolio_id'], + 'transaction_type' => $transaction['transaction_type'], + 'quantity' => $transaction['quantity'], + 'cost_basis' => $transaction['cost_basis'] ?? 0, + 'sale_price' => $transaction['sale_price'], + 'split' => boolval($transaction['split']) ? 1 : 0, + 'reinvested_dividend' => boolval($transaction['reinvested_dividend']) ? 1 : 0, + 'date' => Carbon::parse($transaction['date'])->format('Y-m-d') + ]); + + // stub out related holding + Holding::firstOrCreate([ + 'symbol' => $transaction->symbol, + 'portfolio_id' => $transaction->portfolio_id + ], [ + 'quantity' => 0, + 'average_cost_basis' => 0, + ]); + + return $transaction; + } + + public function batchSize(): int + { + return 150; + } + + public function uniqueBy() + { + return 'id'; } public function rules(): array @@ -54,7 +83,7 @@ class TransactionsSheet implements ToCollection, WithHeadingRow, WithValidation, return [ 'transaction_id' => ['sometimes', 'nullable'], 'symbol' => ['required', 'string'], - 'portfolio_id' => ['required', 'exists:portfolios,id'], + 'portfolio_id' => ['required', new PortfolioAccessValidationRule($this->backupImport->user_id)], 'quantity' => ['required', 'min:0', 'numeric'], 'transaction_type' => ['required', 'in:BUY,SELL'], 'date' => ['required', 'date'], @@ -65,9 +94,4 @@ class TransactionsSheet implements ToCollection, WithHeadingRow, WithValidation, 'sale_price' => ['sometimes', 'nullable', 'min:0', 'numeric'], ]; } - - public function chunkSize(): int - { - return 500; - } } diff --git a/app/Imports/ValidatesPortfolioPermissions.php b/app/Imports/ValidatesPortfolioPermissions.php deleted file mode 100644 index ece5f4a..0000000 --- a/app/Imports/ValidatesPortfolioPermissions.php +++ /dev/null @@ -1,25 +0,0 @@ -user()->portfolios->pluck('id'); - - $collection->pluck('portfolio_id')->unique()->each(function($portfolio) use ($portfolios) { - - if ( - !$portfolios->contains($portfolio) - || auth()->user()->cannot('fullAccess', Portfolio::find($portfolio)) - ) { - - throw new Exception('You do not have permission to access that portfolio.'); - } - }); - } -} diff --git a/app/Jobs/BackupImportJob.php b/app/Jobs/BackupImportJob.php new file mode 100644 index 0000000..05d2200 --- /dev/null +++ b/app/Jobs/BackupImportJob.php @@ -0,0 +1,64 @@ +backupImport), $this->backupImport->path, config('livewire.temporary_file_upload.disk', null)); + } + + /** + * Handle a job failure. + */ + public function failed(?Throwable $e): void + { + $this->backupImport->update([ + 'status' => 'failed', + 'message' => 'Error: '. substr($e->getMessage(), 0, 220), + 'has_errors' => true, + 'completed_at' => now() + ]); + } +} diff --git a/app/Models/BackupImport.php b/app/Models/BackupImport.php new file mode 100644 index 0000000..e59b048 --- /dev/null +++ b/app/Models/BackupImport.php @@ -0,0 +1,52 @@ +status = 'pending'; + $import->message = __('Import starting...'); + }); + + static::created(function ($import) { + + BackupImportJob::dispatch($import); + }); + } + + protected $hidden = []; + + protected $appends = []; + + protected function casts(): array + { + return [ + 'completed_at' => 'datetime' + ]; + } +} diff --git a/app/Models/Portfolio.php b/app/Models/Portfolio.php index 89f2273..194b18b 100644 --- a/app/Models/Portfolio.php +++ b/app/Models/Portfolio.php @@ -69,10 +69,10 @@ class Portfolio extends Model }); } - public function scopeFullAccess() + public function scopeFullAccess($query, $user_id = null) { - return $this->whereHas('users', function ($query) { - $query->where('user_id', auth()->user()->id) + return $query->whereHas('users', function ($query) use ($user_id) { + $query->where('user_id', $user_id ?? auth()->user()->id) ->where(function ($query) { $query->where('full_access', true) ->orWhere('owner', true); diff --git a/app/Rules/PortfolioAccessValidationRule.php b/app/Rules/PortfolioAccessValidationRule.php new file mode 100644 index 0000000..338927b --- /dev/null +++ b/app/Rules/PortfolioAccessValidationRule.php @@ -0,0 +1,27 @@ +user_id)->where('id', $value)->count()) { + $fail(__('You do not have access to that portfolio.')); + } + } +} diff --git a/database/migrations/2024_10_23_000001_create_import_jobs_table.php b/database/migrations/2024_10_23_000001_create_import_jobs_table.php new file mode 100644 index 0000000..7c5ef49 --- /dev/null +++ b/database/migrations/2024_10_23_000001_create_import_jobs_table.php @@ -0,0 +1,34 @@ +uuid('id')->primary(); + $table->foreignIdFor(User::class, 'user_id')->constrained()->onDelete('cascade'); + $table->string('path'); + $table->string('status'); + $table->string('message'); + $table->boolean('has_errors')->default(false); + $table->dateTime('completed_at')->nullable(); + $table->timestamps(); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::dropIfExists('backup_import_jobs'); + } +}; \ No newline at end of file diff --git a/lang/en.json b/lang/en.json index 882fc31..06bd25e 100644 --- a/lang/en.json +++ b/lang/en.json @@ -20,6 +20,7 @@ "Yes": "Yes", "you": "you", "Nothing to show here yet": "Nothing to show here yet", + "Try again": "Try again", "Hang on! You're doing that too much.": "Hang on! You're doing that too much.", "Delete Account": "Delete Account", @@ -360,6 +361,14 @@ "Hey again!": "Hey again!", "Before you can get started with Investbrain, let's complete your profile:": "Before you can get started with Investbrain, let's complete your profile:", "Or login with SSO:": "Or login with SSO:", - "Create Password": "Create Password" + "Create Password": "Create Password", + "You do not have access to that portfolio.": "You do not have access to that portfolio.", + "Import starting...": "Import starting...", + "Import is in progress...": "Import is in progress...", + "Importing portfolios...": "Importing portfolios...", + "Importing transactions...": "Importing transactions...", + "Importing daily changes...": "Importing daily changes...", + "Import completed successfully!": "Import completed successfully!", + "Your import will continue in the background.": "Your import will continue in the background." } \ No newline at end of file diff --git a/lang/es.json b/lang/es.json index c89d3b2..275982f 100644 --- a/lang/es.json +++ b/lang/es.json @@ -20,6 +20,7 @@ "Yes": "Sí", "you": "tú", "Nothing to show here yet": "No hay nada que mostrar aquí todavía", + "Try again": "Intentar otra vez", "Hang on! You're doing that too much.": "¡Por favor espere un momento!", "Delete Account": "Eliminar Cuenta", @@ -360,5 +361,14 @@ "Hey again!": "¡Oye de nuevo!", "Before you can get started with Investbrain, let's complete your profile:": "Antes de poder comenzar a utilizar Investbrain, deberá crear una cuenta:", "Or login with SSO:": "O iniciar sesión mediante SSO:", - "Create Password": "Crear Contraseña" + "Create Password": "Crear Contraseña", + + "You do not have access to that portfolio.": "No tienes acceso a ese portafolio.", + "Import starting...": "Iniciando la importación...", + "Import is in progress...": "La importación está en progreso...", + "Importing portfolios...": "Importando portafolios...", + "Importing transactions...": "Importando transacciones...", + "Importing daily changes...": "Importando cambios diarios...", + "Import completed successfully!": "¡La importación se completó con éxito!", + "Your import will continue in the background.": "La importación continuará en segundo plano." } \ No newline at end of file diff --git a/resources/views/components/confirmation-modal.blade.php b/resources/views/components/confirmation-modal.blade.php index 1f599da..fe108ed 100644 --- a/resources/views/components/confirmation-modal.blade.php +++ b/resources/views/components/confirmation-modal.blade.php @@ -21,7 +21,7 @@ -
+
{{ $footer }}
diff --git a/resources/views/components/dialog-modal.blade.php b/resources/views/components/dialog-modal.blade.php index d901537..0d06a43 100644 --- a/resources/views/components/dialog-modal.blade.php +++ b/resources/views/components/dialog-modal.blade.php @@ -11,7 +11,7 @@
-
+
{{ $footer }}
diff --git a/resources/views/livewire/import-portfolios-field.blade.php b/resources/views/livewire/import-portfolios-field.blade.php index 720436e..bde3590 100644 --- a/resources/views/livewire/import-portfolios-field.blade.php +++ b/resources/views/livewire/import-portfolios-field.blade.php @@ -3,6 +3,7 @@ use Livewire\WithFileUploads; use Livewire\Volt\Component; use Mary\Traits\Toast; +use App\Models\BackupImport as BackupImportModel; use App\Imports\BackupImport; use App\Exports\BackupExport; use Livewire\Attributes\Rule; @@ -16,6 +17,10 @@ new class extends Component { #[Rule('required|extensions:xlsx|mimes:xlsx|max:2048')] public $file; + public bool $importStatusDialog = false; + public ?BackupImportModel $backupImport = null; + public int $percent = 10; + // methods public function import() { @@ -27,20 +32,45 @@ new class extends Component { return; } - try { + $this->backupImport = BackupImportModel::create([ + 'user_id' => auth()->user()->id, + 'path' => $this->file->getPathname() + ]); - $import = Excel::import( - new BackupImport, - $this->file->getPathname(), - config('livewire.temporary_file_upload.disk', null) - ); + $this->importStatusDialog = true; - } catch (\Throwable $e) { - - return $this->error($e->getMessage()); + } + + public function checkImportStatus() + { + if (Str::contains($this->backupImport?->message, 'portfolios')) { + + $this->percent = (1/2) * 100; } - $this->success(__('Successfully imported!'), redirectTo: route('dashboard')); + if (Str::contains($this->backupImport?->message, 'transactions')) { + + $this->percent = (3/4) * 100; + } + + if (Str::contains($this->backupImport?->message, 'daily changes')) { + + $this->percent = (7/8) * 100; + } + + if ($this->backupImport?->status == 'failed') { + + unset($this->file); + $this->percent = 100; + } + + if ($this->backupImport?->status == 'success') { + + $this->importStatusDialog = false; + $this->backupImport = null; + + $this->success(__('Successfully imported!'), redirectTo: route('dashboard')); + } } public function downloadTemplate() @@ -60,13 +90,47 @@ new class extends Component { {{ __('Download import template.') }} - +
-
+ + + + @if($backupImport?->status) +
+ {{ $backupImport?->message }} +
+ @endif +
+ + @if($backupImport?->status != 'failed') + + @endif + + + + @if($backupImport?->status == 'failed') + + {{ __('Try again') }} + @else +
{{ __('Your import will continue in the background.') }}
+ + {{ __('Close') }} + @endif +
+
+ +