javier/investbrain
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat:improved background importing

Browse Source
This commit is contained in:
hackerESQ
2024-10-24 14:48:24 -05:00
parent 7543c0a865
commit 3c368310ad
15 changed files with 459 additions and 139 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/Imports/BackupImport.php
+37 -7
View File
@@ -4,13 +4,17 @@ namespace App\Imports;
use App\Imports\Sheets\PortfoliosSheet;
use Illuminate\Support\Facades\Artisan;
use App\Console\Commands\SyncHoldingData;
use App\Imports\Sheets\DailyChangesSheet;
use App\Imports\Sheets\TransactionsSheet;
use Maatwebsite\Excel\Events\AfterImport;
use Maatwebsite\Excel\Concerns\Importable;
use Maatwebsite\Excel\Concerns\WithEvents;
use Maatwebsite\Excel\Events\BeforeImport;
use Maatwebsite\Excel\Events\ImportFailed;
use App\Console\Commands\RefreshMarketData;
use App\Console\Commands\RefreshDividendData;
use App\Models\BackupImport as BackupImportModel;
use Maatwebsite\Excel\Concerns\WithMultipleSheets;
class BackupImport implements WithMultipleSheets, WithEvents
@@ -18,25 +22,51 @@ class BackupImport implements WithMultipleSheets, WithEvents
use Importable;
public function __construct(
public BackupImportModel $backupImportModel
) { }
/**
* @return array
*/
public function registerEvents(): array
{
return [
AfterImport::class =>
fn() => Artisan::queue(RefreshMarketData::class, ['--force' => true])->chain([
fn() => Artisan::call(RefreshDividendData::class, ['--force' => true])
])
BeforeImport::class => fn() => $this->backupImportModel->update([
'status' => 'in_progress',
'message' => __('Import is in progress...'),
]),
AfterImport::class => function () {
$this->backupImportModel->update([
'status' => 'success',
'message' => 'Import completed successfully!',
'completed_at' => now()
]);
Artisan::queue(RefreshMarketData::class, ['--user' => $this->backupImportModel->user_id, '--force' => true])
->chain([
fn() => Artisan::call(RefreshDividendData::class, ['--user' => $this->backupImportModel->user_id, '--force' => true])
])
->chain([
fn() => Artisan::call(SyncHoldingData::class, ['--user' => $this->backupImportModel->user_id])
]);
},
ImportFailed::class => fn(ImportFailed $event) => $this->backupImportModel->update([
'status' => 'failed',
'message' => 'Error: '. substr($event->getException()->getMessage(), 0, 220),
'has_errors' => true,
'completed_at' => now()
]),
];
}
public function sheets(): array
{
return [
'Portfolios' => new PortfoliosSheet,
'Transactions' => new TransactionsSheet,
'Daily Changes' => new DailyChangesSheet,
'Portfolios' => new PortfoliosSheet($this->backupImportModel),
'Transactions' => new TransactionsSheet($this->backupImportModel),
'Daily Changes' => new DailyChangesSheet($this->backupImportModel),
];
}
}
app/Imports/Sheets/DailyChangesSheet.php
+49 -42
View File
@@ -3,57 +3,69 @@
namespace App\Imports\Sheets;
use App\Models\DailyChange;
use App\Models\BackupImport;
use Illuminate\Support\Carbon;
use Illuminate\Support\Collection;
use Maatwebsite\Excel\Concerns\ToCollection;
use App\Imports\ValidatesPortfolioPermissions;
use Illuminate\Support\Facades\DB;
use Maatwebsite\Excel\Concerns\ToModel;
use Maatwebsite\Excel\Events\BeforeSheet;
use Maatwebsite\Excel\Concerns\WithEvents;
use Maatwebsite\Excel\Concerns\WithUpserts;
use App\Rules\PortfolioAccessValidationRule;
use Maatwebsite\Excel\Concerns\SkipsEmptyRows;
use Maatwebsite\Excel\Concerns\WithHeadingRow;
use Maatwebsite\Excel\Concerns\WithValidation;
use Maatwebsite\Excel\Concerns\WithChunkReading;
use Maatwebsite\Excel\Concerns\WithBatchInserts;
class DailyChangesSheet implements ToCollection, WithHeadingRow, WithValidation, SkipsEmptyRows, WithChunkReading
class DailyChangesSheet implements ToModel, WithHeadingRow, WithValidation, WithBatchInserts, WithUpserts, SkipsEmptyRows, WithEvents
{
use ValidatesPortfolioPermissions;
public function __construct(
public BackupImport $backupImport
) { }
public function collection(Collection $dailyChanges)
/**
* @return array
*/
public function registerEvents(): array
{
$this->validatePortfolioPermissions($dailyChanges);
return [
BeforeSheet::class => function(BeforeSheet $event) {
DB::commit();
$this->backupImport->update([
'message' => __('Importing daily changes...'),
]);
DB::beginTransaction();
}
];
}
public function model(array $dailyChange)
{
return new DailyChange([
'total_market_value' => $dailyChange['total_market_value'],
'total_cost_basis' => $dailyChange['total_cost_basis'],
'total_gain' => $dailyChange['total_gain'],
'total_dividends_earned' => $dailyChange['total_dividends_earned'],
'realized_gains' => $dailyChange['realized_gains'],
'annotation' => $dailyChange['annotation'],
'portfolio_id' => $dailyChange['portfolio_id'],
'date' => Carbon::parse($dailyChange['date'])->format('Y-m-d')
]);
}
public function batchSize(): int
{
return 150;
}
$chunkSize = 500;
$dailyChanges->chunk($chunkSize)->each(function ($chunk) {
// have to manually format dates since we're doing a raw upsert
$chunk = $chunk->map(function ($daily) {
$daily['date'] = Carbon::parse($daily['date'])->format('Y-m-d');
return $daily;
});
DailyChange::upsert(
$chunk->toArray(),
[
'portfolio_id',
'date'
],
[
'total_market_value',
'total_cost_basis',
'total_gain',
'total_dividends_earned',
'realized_gains',
'annotation'
]
);
});
public function uniqueBy()
{
return ['portfolio_id', 'date'];
}
public function rules(): array
{
return [
'portfolio_id' => ['required', 'exists:portfolios,id'],
'portfolio_id' => ['required', new PortfolioAccessValidationRule($this->backupImport->user_id)],
'date' => ['required', 'date'],
'total_market_value' => ['sometimes', 'nullable', 'numeric'],
'total_cost_basis' => ['sometimes', 'nullable', 'min:0', 'numeric'],
@@ -63,9 +75,4 @@ class DailyChangesSheet implements ToCollection, WithHeadingRow, WithValidation,
'annotation' => ['sometimes', 'nullable', 'string'],
];
}
public function chunkSize(): int
{
return 500;
}
}
app/Imports/Sheets/PortfoliosSheet.php
+28 -4
View File
@@ -2,24 +2,48 @@
namespace App\Imports\Sheets;
use App\Console\Commands\SyncDailyChange;
use App\Models\Portfolio;
use App\Models\BackupImport;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Artisan;
use App\Console\Commands\SyncDailyChange;
use Maatwebsite\Excel\Events\BeforeSheet;
use Maatwebsite\Excel\Concerns\WithEvents;
use Maatwebsite\Excel\Concerns\ToCollection;
use Maatwebsite\Excel\Concerns\SkipsEmptyRows;
use Maatwebsite\Excel\Concerns\WithHeadingRow;
use Maatwebsite\Excel\Concerns\WithValidation;
class PortfoliosSheet implements ToCollection, WithValidation, WithHeadingRow, SkipsEmptyRows
class PortfoliosSheet implements ToCollection, WithValidation, WithHeadingRow, SkipsEmptyRows, WithEvents
{
public function __construct(
public BackupImport $backupImport
) { }
/**
* @return array
*/
public function registerEvents(): array
{
return [
BeforeSheet::class => function(BeforeSheet $event) {
DB::commit();
$this->backupImport->update([
'message' => __('Importing portfolios...'),
]);
DB::beginTransaction();
}
];
}
public function collection(Collection $portfolios)
{
foreach ($portfolios as $index => $portfolio) {
Portfolio::unguard();
$portfolio = Portfolio::updateOrCreate([
$portfolio = Portfolio::fullAccess($this->backupImport->user_id)->updateOrCreate([
'id' => $portfolio['portfolio_id']
], [
'id' => $portfolio['portfolio_id'] ?? null,
app/Imports/Sheets/TransactionsSheet.php
+66 -42
View File
@@ -2,51 +2,80 @@
namespace App\Imports\Sheets;
use App\Models\Holding;
use App\Models\Transaction;
use Illuminate\Support\Collection;
use Maatwebsite\Excel\Concerns\ToCollection;
use App\Imports\ValidatesPortfolioPermissions;
use Illuminate\Support\Str;
use App\Models\BackupImport;
use Illuminate\Support\Carbon;
use Illuminate\Support\Facades\DB;
use Maatwebsite\Excel\Concerns\ToModel;
use Maatwebsite\Excel\Events\BeforeSheet;
use Maatwebsite\Excel\Concerns\WithUpserts;
use App\Rules\PortfolioAccessValidationRule;
use Maatwebsite\Excel\Concerns\SkipsEmptyRows;
use Maatwebsite\Excel\Concerns\WithHeadingRow;
use Maatwebsite\Excel\Concerns\WithValidation;
use Maatwebsite\Excel\Concerns\WithChunkReading;
use Maatwebsite\Excel\Concerns\WithBatchInserts;
use Maatwebsite\Excel\Concerns\WithEvents;
class TransactionsSheet implements ToCollection, WithHeadingRow, WithValidation, SkipsEmptyRows, WithChunkReading
class TransactionsSheet implements ToModel, WithHeadingRow, WithValidation, WithBatchInserts, WithUpserts, SkipsEmptyRows, WithEvents
{
use ValidatesPortfolioPermissions;
public function collection(Collection $transactions)
public function __construct(
public BackupImport $backupImport
) { }
/**
* @return array
*/
public function registerEvents(): array
{
$this->validatePortfolioPermissions($transactions);
Transaction::withoutEvents(function () use ($transactions) {
foreach ($transactions->sortBy('date') as $transaction) {
Transaction::where('id', $transaction['transaction_id'])
->firstOr(function () use ($transaction) {
$transaction = Transaction::make()->forceFill([
'id' => $transaction['transaction_id'],
'symbol' => $transaction['symbol'],
'portfolio_id' => $transaction['portfolio_id'],
'transaction_type' => $transaction['transaction_type'],
'quantity' => $transaction['quantity'],
'cost_basis' => $transaction['cost_basis'] ?? 0,
'sale_price' => $transaction['sale_price'],
'split' => $transaction['split'] ?? null,
'reinvested_dividend' => $transaction['reinvested_dividend'] ?? null,
'date' => $transaction['date'],
]);
$transaction->save();
return $transaction;
})
->syncToHolding();
return [
BeforeSheet::class => function(BeforeSheet $event) {
DB::commit();
$this->backupImport->update([
'message' => __('Importing transactions...'),
]);
DB::beginTransaction();
}
});
];
}
public function model(array $transaction)
{
$transaction = new Transaction([
'id' => $transaction['transaction_id'] ?? Str::uuid()->toString(),
'symbol' => strtoupper($transaction['symbol']),
'portfolio_id' => $transaction['portfolio_id'],
'transaction_type' => $transaction['transaction_type'],
'quantity' => $transaction['quantity'],
'cost_basis' => $transaction['cost_basis'] ?? 0,
'sale_price' => $transaction['sale_price'],
'split' => boolval($transaction['split']) ? 1 : 0,
'reinvested_dividend' => boolval($transaction['reinvested_dividend']) ? 1 : 0,
'date' => Carbon::parse($transaction['date'])->format('Y-m-d')
]);
// stub out related holding
Holding::firstOrCreate([
'symbol' => $transaction->symbol,
'portfolio_id' => $transaction->portfolio_id
], [
'quantity' => 0,
'average_cost_basis' => 0,
]);
return $transaction;
}
public function batchSize(): int
{
return 150;
}
public function uniqueBy()
{
return 'id';
}
public function rules(): array
@@ -54,7 +83,7 @@ class TransactionsSheet implements ToCollection, WithHeadingRow, WithValidation,
return [
'transaction_id' => ['sometimes', 'nullable'],
'symbol' => ['required', 'string'],
'portfolio_id' => ['required', 'exists:portfolios,id'],
'portfolio_id' => ['required', new PortfolioAccessValidationRule($this->backupImport->user_id)],
'quantity' => ['required', 'min:0', 'numeric'],
'transaction_type' => ['required', 'in:BUY,SELL'],
'date' => ['required', 'date'],
@@ -65,9 +94,4 @@ class TransactionsSheet implements ToCollection, WithHeadingRow, WithValidation,
'sale_price' => ['sometimes', 'nullable', 'min:0', 'numeric'],
];
}
public function chunkSize(): int
{
return 500;
}
}
app/Imports/ValidatesPortfolioPermissions.php
-25
View File
@@ -1,25 +0,0 @@
<?php
namespace App\Imports;
use Exception;
use App\Models\Portfolio;
trait ValidatesPortfolioPermissions {
public function validatePortfolioPermissions($collection)
{
$portfolios = auth()->user()->portfolios->pluck('id');
$collection->pluck('portfolio_id')->unique()->each(function($portfolio) use ($portfolios) {
if (
!$portfolios->contains($portfolio)
|| auth()->user()->cannot('fullAccess', Portfolio::find($portfolio))
) {
throw new Exception('You do not have permission to access that portfolio.');
}
});
}
}
app/Jobs/BackupImportJob.php
+64
View File
@@ -0,0 +1,64 @@
<?php
namespace App\Jobs;
use Throwable;
use App\Models\User;
use App\Models\BackupImport;
use Illuminate\Support\Facades\Auth;
use Maatwebsite\Excel\Facades\Excel;
use Illuminate\Foundation\Queue\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use App\Imports\BackupImport as BackupImportExcel;
class BackupImportJob implements ShouldQueue
{
use Queueable;
/**
* The number of times the job may be attempted.
*/
public $tries = 1;
/**
* The number of seconds the job can run before timing out.
*
* @var int
*/
public $timeout = 300;
/**
* Indicate if the job should be marked as failed on timeout.
*
* @var bool
*/
public $failOnTimeout = true;
/**
* Create a new job instance.
*/
public function __construct(
public BackupImport $backupImport
) { }
/**
* Execute the job.
*/
public function handle(): void
{
Excel::import(new BackupImportExcel($this->backupImport), $this->backupImport->path, config('livewire.temporary_file_upload.disk', null));
}
/**
* Handle a job failure.
*/
public function failed(?Throwable $e): void
{
$this->backupImport->update([
'status' => 'failed',
'message' => 'Error: '. substr($e->getMessage(), 0, 220),
'has_errors' => true,
'completed_at' => now()
]);
}
}
app/Models/BackupImport.php
+52
View File
@@ -0,0 +1,52 @@
<?php
namespace App\Models;
use Maatwebsite\Excel\Facades\Excel;
use Illuminate\Database\Eloquent\Model;
use App\Imports\BackupImport as BackupImportExcel;
use App\Jobs\BackupImportJob;
use Illuminate\Database\Eloquent\Concerns\HasUuids;
class BackupImport extends Model
{
use HasUuids;
protected $table = 'backup_import_jobs';
protected $fillable = [
'user_id',
'path',
'status', // pending, in_progress, success, failed
'message', // Import starting, Import is in progress, Importing portfolios, Importing transactions, Importing daily changes, Import completed successfully
'has_errors',
'completed_at'
];
protected static function boot()
{
parent::boot();
static::creating(function ($import) {
$import->status = 'pending';
$import->message = __('Import starting...');
});
static::created(function ($import) {
BackupImportJob::dispatch($import);
});
}
protected $hidden = [];
protected $appends = [];
protected function casts(): array
{
return [
'completed_at' => 'datetime'
];
}
}
app/Models/Portfolio.php
+3 -3
View File
@@ -69,10 +69,10 @@ class Portfolio extends Model
});
}
public function scopeFullAccess()
public function scopeFullAccess($query, $user_id = null)
{
return $this->whereHas('users', function ($query) {
$query->where('user_id', auth()->user()->id)
return $query->whereHas('users', function ($query) use ($user_id) {
$query->where('user_id', $user_id ?? auth()->user()->id)
->where(function ($query) {
$query->where('full_access', true)
->orWhere('owner', true);
app/Rules/PortfolioAccessValidationRule.php
+27
View File
@@ -0,0 +1,27 @@
<?php
namespace App\Rules;
use App\Models\Portfolio;
use Illuminate\Contracts\Validation\ValidationRule;
class PortfolioAccessValidationRule implements ValidationRule
{
public function __construct(
public ?string $user_id
) { }
/**
* Validate the attribute.
*
* @param string $attribute
* @param mixed $value
* @param \Closure $fail
* @return void
*/
public function validate(string $attribute, mixed $value, \Closure $fail): void
{
if (!Portfolio::fullAccess($this->user_id)->where('id', $value)->count()) {
$fail(__('You do not have access to that portfolio.'));
}
}
}
database/migrations/2024_10_23_000001_create_import_jobs_table.php
+34
View File
@@ -0,0 +1,34 @@
<?php
use App\Models\User;
use Illuminate\Support\Facades\Schema;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Database\Migrations\Migration;
return new class extends Migration
{
/**
* Run the migrations.
*/
public function up(): void
{
Schema::create('backup_import_jobs', function (Blueprint $table) {
$table->uuid('id')->primary();
$table->foreignIdFor(User::class, 'user_id')->constrained()->onDelete('cascade');
$table->string('path');
$table->string('status');
$table->string('message');
$table->boolean('has_errors')->default(false);
$table->dateTime('completed_at')->nullable();
$table->timestamps();
});
}
/**
* Reverse the migrations.
*/
public function down(): void
{
Schema::dropIfExists('backup_import_jobs');
}
};
lang/en.json
+10 -1
View File
@@ -20,6 +20,7 @@
"Yes": "Yes",
"you": "you",
"Nothing to show here yet": "Nothing to show here yet",
"Try again": "Try again",
"Hang on! You're doing that too much.": "Hang on! You're doing that too much.",
"Delete Account": "Delete Account",
@@ -360,6 +361,14 @@
"Hey again!": "Hey again!",
"Before you can get started with Investbrain, let's complete your profile:": "Before you can get started with Investbrain, let's complete your profile:",
"Or login with SSO:": "Or login with SSO:",
"Create Password": "Create Password"
"Create Password": "Create Password",
"You do not have access to that portfolio.": "You do not have access to that portfolio.",
"Import starting...": "Import starting...",
"Import is in progress...": "Import is in progress...",
"Importing portfolios...": "Importing portfolios...",
"Importing transactions...": "Importing transactions...",
"Importing daily changes...": "Importing daily changes...",
"Import completed successfully!": "Import completed successfully!",
"Your import will continue in the background.": "Your import will continue in the background."
}
lang/es.json
+11 -1
View File
@@ -20,6 +20,7 @@
"Yes": "Sí",
"you": "tú",
"Nothing to show here yet": "No hay nada que mostrar aquí todavía",
"Try again": "Intentar otra vez",
"Hang on! You're doing that too much.": "¡Por favor espere un momento!",
"Delete Account": "Eliminar Cuenta",
@@ -360,5 +361,14 @@
"Hey again!": "¡Oye de nuevo!",
"Before you can get started with Investbrain, let's complete your profile:": "Antes de poder comenzar a utilizar Investbrain, deberá crear una cuenta:",
"Or login with SSO:": "O iniciar sesión mediante SSO:",
"Create Password": "Crear Contraseña"
"Create Password": "Crear Contraseña",
"You do not have access to that portfolio.": "No tienes acceso a ese portafolio.",
"Import starting...": "Iniciando la importación...",
"Import is in progress...": "La importación está en progreso...",
"Importing portfolios...": "Importando portafolios...",
"Importing transactions...": "Importando transacciones...",
"Importing daily changes...": "Importando cambios diarios...",
"Import completed successfully!": "¡La importación se completó con éxito!",
"Your import will continue in the background.": "La importación continuará en segundo plano."
}
resources/views/components/confirmation-modal.blade.php
+1 -1
View File
@@ -21,7 +21,7 @@
</div>
</div>
<div class="flex flex-row justify-end mt-3 p-2 text-end">
<div class="flex flex-row items-center justify-end mt-3 p-2 text-end">
{{ $footer }}
</div>
</x-ib-livewire-modal>
resources/views/components/dialog-modal.blade.php
+1 -1
View File
@@ -11,7 +11,7 @@
</div>
</div>
<div class="flex flex-row justify-end mt-3 p-2 text-end">
<div class="flex flex-row items-center justify-end mt-3 p-2 text-end">
{{ $footer }}
</div>
</x-ib-livewire-modal>
resources/views/livewire/import-portfolios-field.blade.php
+76 -12
View File
@@ -3,6 +3,7 @@
use Livewire\WithFileUploads;
use Livewire\Volt\Component;
use Mary\Traits\Toast;
use App\Models\BackupImport as BackupImportModel;
use App\Imports\BackupImport;
use App\Exports\BackupExport;
use Livewire\Attributes\Rule;
@@ -16,6 +17,10 @@ new class extends Component {
#[Rule('required|extensions:xlsx|mimes:xlsx|max:2048')]
public $file;
public bool $importStatusDialog = false;
public ?BackupImportModel $backupImport = null;
public int $percent = 10;
// methods
public function import()
{
@@ -27,20 +32,45 @@ new class extends Component {
return;
}
try {
$this->backupImport = BackupImportModel::create([
'user_id' => auth()->user()->id,
'path' => $this->file->getPathname()
]);
$import = Excel::import(
new BackupImport,
$this->file->getPathname(),
config('livewire.temporary_file_upload.disk', null)
);
$this->importStatusDialog = true;
} catch (\Throwable $e) {
return $this->error($e->getMessage());
}
public function checkImportStatus()
{
if (Str::contains($this->backupImport?->message, 'portfolios')) {
$this->percent = (1/2) * 100;
}
$this->success(__('Successfully imported!'), redirectTo: route('dashboard'));
if (Str::contains($this->backupImport?->message, 'transactions')) {
$this->percent = (3/4) * 100;
}
if (Str::contains($this->backupImport?->message, 'daily changes')) {
$this->percent = (7/8) * 100;
}
if ($this->backupImport?->status == 'failed') {
unset($this->file);
$this->percent = 100;
}
if ($this->backupImport?->status == 'success') {
$this->importStatusDialog = false;
$this->backupImport = null;
$this->success(__('Successfully imported!'), redirectTo: route('dashboard'));
}
}
public function downloadTemplate()
@@ -60,13 +90,47 @@ new class extends Component {
<strong><a href="#" title="{{ __('Click to download import template.') }}" @click="$wire.downloadTemplate()"> {{ __('Download import template.') }}</a></strong>
</x-slot>
<x-slot name="form">
<x-slot:form>
<div class="col-span-6 sm:col-span-4">
<x-file wire:model="file" label="{{ __('Select a file') }}" hint="" accept=".xlsx" required />
</div>
</x-slot>
<x-dialog-modal wire:model.live="importStatusDialog" persistent>
<x-slot name="title">
@if($backupImport?->status)
<div
class="{{ $backupImport?->status == 'failed' ? 'text-error' : '' }}"
>
{{ $backupImport?->message }}
</div>
@endif
</x-slot>
<x-slot name="content">
@if($backupImport?->status != 'failed')
<x-progress
:indeterminate="$backupImport?->status == 'pending'"
class="progress-primary h-3"
value="{{ $percent }}"
max="100"
/>
@endif
</x-slot>
<x-slot name="footer">
@if($backupImport?->status == 'failed')
<x-button wire:click="$toggle('importStatusDialog')"> {{ __('Try again') }} </x-button>
@else
<div wire:poll="checkImportStatus" class="text-gray-400 text-sm">{{ __('Your import will continue in the background.') }}</div>
<x-ib-flex-spacer />
<x-button wire:click="$toggle('importStatusDialog')"> {{ __('Close') }} </x-button>
@endif
</x-slot>
</x-dialog-modal>
</x-slot:form>
<x-slot name="actions">