javier/investbrain
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat:improved background importing

Browse Source
This commit is contained in:
hackerESQ
2024-10-24 14:48:24 -05:00
parent 7543c0a865
commit 3c368310ad
15 changed files with 459 additions and 139 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/Imports/BackupImport.php
+37 -7
View File
@@ -4,13 +4,17 @@ namespace App\Imports;
use App\Imports\Sheets\PortfoliosSheet;
use Illuminate\Support\Facades\Artisan;
use App\Console\Commands\SyncHoldingData;
use App\Imports\Sheets\DailyChangesSheet;
use App\Imports\Sheets\TransactionsSheet;
use Maatwebsite\Excel\Events\AfterImport;
use Maatwebsite\Excel\Concerns\Importable;
use Maatwebsite\Excel\Concerns\WithEvents;
use Maatwebsite\Excel\Events\BeforeImport;
use Maatwebsite\Excel\Events\ImportFailed;
use App\Console\Commands\RefreshMarketData;
use App\Console\Commands\RefreshDividendData;
use App\Models\BackupImport as BackupImportModel;
use Maatwebsite\Excel\Concerns\WithMultipleSheets;
class BackupImport implements WithMultipleSheets, WithEvents
@@ -18,25 +22,51 @@ class BackupImport implements WithMultipleSheets, WithEvents
use Importable;
public function __construct(
public BackupImportModel $backupImportModel
) { }
/**
* @return array
*/
public function registerEvents(): array
{
return [
AfterImport::class =>
fn() => Artisan::queue(RefreshMarketData::class, ['--force' => true])->chain([
fn() => Artisan::call(RefreshDividendData::class, ['--force' => true])
])
BeforeImport::class => fn() => $this->backupImportModel->update([
'status' => 'in_progress',
'message' => __('Import is in progress...'),
]),
AfterImport::class => function () {
$this->backupImportModel->update([
'status' => 'success',
'message' => 'Import completed successfully!',
'completed_at' => now()
]);
Artisan::queue(RefreshMarketData::class, ['--user' => $this->backupImportModel->user_id, '--force' => true])
->chain([
fn() => Artisan::call(RefreshDividendData::class, ['--user' => $this->backupImportModel->user_id, '--force' => true])
])
->chain([
fn() => Artisan::call(SyncHoldingData::class, ['--user' => $this->backupImportModel->user_id])
]);
},
ImportFailed::class => fn(ImportFailed $event) => $this->backupImportModel->update([
'status' => 'failed',
'message' => 'Error: '. substr($event->getException()->getMessage(), 0, 220),
'has_errors' => true,
'completed_at' => now()
]),
];
}
public function sheets(): array
{
return [
'Portfolios' => new PortfoliosSheet,
'Transactions' => new TransactionsSheet,
'Daily Changes' => new DailyChangesSheet,
'Portfolios' => new PortfoliosSheet($this->backupImportModel),
'Transactions' => new TransactionsSheet($this->backupImportModel),
'Daily Changes' => new DailyChangesSheet($this->backupImportModel),
];
}
}
app/Imports/Sheets/DailyChangesSheet.php
+49 -42
View File
@@ -3,57 +3,69 @@
namespace App\Imports\Sheets;
use App\Models\DailyChange;
use App\Models\BackupImport;
use Illuminate\Support\Carbon;
use Illuminate\Support\Collection;
use Maatwebsite\Excel\Concerns\ToCollection;
use App\Imports\ValidatesPortfolioPermissions;
use Illuminate\Support\Facades\DB;
use Maatwebsite\Excel\Concerns\ToModel;
use Maatwebsite\Excel\Events\BeforeSheet;
use Maatwebsite\Excel\Concerns\WithEvents;
use Maatwebsite\Excel\Concerns\WithUpserts;
use App\Rules\PortfolioAccessValidationRule;
use Maatwebsite\Excel\Concerns\SkipsEmptyRows;
use Maatwebsite\Excel\Concerns\WithHeadingRow;
use Maatwebsite\Excel\Concerns\WithValidation;
use Maatwebsite\Excel\Concerns\WithChunkReading;
use Maatwebsite\Excel\Concerns\WithBatchInserts;
class DailyChangesSheet implements ToCollection, WithHeadingRow, WithValidation, SkipsEmptyRows, WithChunkReading
class DailyChangesSheet implements ToModel, WithHeadingRow, WithValidation, WithBatchInserts, WithUpserts, SkipsEmptyRows, WithEvents
{
use ValidatesPortfolioPermissions;
public function __construct(
public BackupImport $backupImport
) { }
public function collection(Collection $dailyChanges)
/**
* @return array
*/
public function registerEvents(): array
{
$this->validatePortfolioPermissions($dailyChanges);
return [
BeforeSheet::class => function(BeforeSheet $event) {
DB::commit();
$this->backupImport->update([
'message' => __('Importing daily changes...'),
]);
DB::beginTransaction();
}
];
}
public function model(array $dailyChange)
{
return new DailyChange([
'total_market_value' => $dailyChange['total_market_value'],
'total_cost_basis' => $dailyChange['total_cost_basis'],
'total_gain' => $dailyChange['total_gain'],
'total_dividends_earned' => $dailyChange['total_dividends_earned'],
'realized_gains' => $dailyChange['realized_gains'],
'annotation' => $dailyChange['annotation'],
'portfolio_id' => $dailyChange['portfolio_id'],
'date' => Carbon::parse($dailyChange['date'])->format('Y-m-d')
]);
}
public function batchSize(): int
{
return 150;
}
$chunkSize = 500;
$dailyChanges->chunk($chunkSize)->each(function ($chunk) {
// have to manually format dates since we're doing a raw upsert
$chunk = $chunk->map(function ($daily) {
$daily['date'] = Carbon::parse($daily['date'])->format('Y-m-d');
return $daily;
});
DailyChange::upsert(
$chunk->toArray(),
[
'portfolio_id',
'date'
],
[
'total_market_value',
'total_cost_basis',
'total_gain',
'total_dividends_earned',
'realized_gains',
'annotation'
]
);
});
public function uniqueBy()
{
return ['portfolio_id', 'date'];
}
public function rules(): array
{
return [
'portfolio_id' => ['required', 'exists:portfolios,id'],
'portfolio_id' => ['required', new PortfolioAccessValidationRule($this->backupImport->user_id)],
'date' => ['required', 'date'],
'total_market_value' => ['sometimes', 'nullable', 'numeric'],
'total_cost_basis' => ['sometimes', 'nullable', 'min:0', 'numeric'],
@@ -63,9 +75,4 @@ class DailyChangesSheet implements ToCollection, WithHeadingRow, WithValidation,
'annotation' => ['sometimes', 'nullable', 'string'],
];
}
public function chunkSize(): int
{
return 500;
}
}
app/Imports/Sheets/PortfoliosSheet.php
+28 -4
View File
@@ -2,24 +2,48 @@
namespace App\Imports\Sheets;
use App\Console\Commands\SyncDailyChange;
use App\Models\Portfolio;
use App\Models\BackupImport;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Artisan;
use App\Console\Commands\SyncDailyChange;
use Maatwebsite\Excel\Events\BeforeSheet;
use Maatwebsite\Excel\Concerns\WithEvents;
use Maatwebsite\Excel\Concerns\ToCollection;
use Maatwebsite\Excel\Concerns\SkipsEmptyRows;
use Maatwebsite\Excel\Concerns\WithHeadingRow;
use Maatwebsite\Excel\Concerns\WithValidation;
class PortfoliosSheet implements ToCollection, WithValidation, WithHeadingRow, SkipsEmptyRows
class PortfoliosSheet implements ToCollection, WithValidation, WithHeadingRow, SkipsEmptyRows, WithEvents
{
public function __construct(
public BackupImport $backupImport
) { }
/**
* @return array
*/
public function registerEvents(): array
{
return [
BeforeSheet::class => function(BeforeSheet $event) {
DB::commit();
$this->backupImport->update([
'message' => __('Importing portfolios...'),
]);
DB::beginTransaction();
}
];
}
public function collection(Collection $portfolios)
{
foreach ($portfolios as $index => $portfolio) {
Portfolio::unguard();
$portfolio = Portfolio::updateOrCreate([
$portfolio = Portfolio::fullAccess($this->backupImport->user_id)->updateOrCreate([
'id' => $portfolio['portfolio_id']
], [
'id' => $portfolio['portfolio_id'] ?? null,
app/Imports/Sheets/TransactionsSheet.php
+66 -42
View File
@@ -2,51 +2,80 @@
namespace App\Imports\Sheets;
use App\Models\Holding;
use App\Models\Transaction;
use Illuminate\Support\Collection;
use Maatwebsite\Excel\Concerns\ToCollection;
use App\Imports\ValidatesPortfolioPermissions;
use Illuminate\Support\Str;
use App\Models\BackupImport;
use Illuminate\Support\Carbon;
use Illuminate\Support\Facades\DB;
use Maatwebsite\Excel\Concerns\ToModel;
use Maatwebsite\Excel\Events\BeforeSheet;
use Maatwebsite\Excel\Concerns\WithUpserts;
use App\Rules\PortfolioAccessValidationRule;
use Maatwebsite\Excel\Concerns\SkipsEmptyRows;
use Maatwebsite\Excel\Concerns\WithHeadingRow;
use Maatwebsite\Excel\Concerns\WithValidation;
use Maatwebsite\Excel\Concerns\WithChunkReading;
use Maatwebsite\Excel\Concerns\WithBatchInserts;
use Maatwebsite\Excel\Concerns\WithEvents;
class TransactionsSheet implements ToCollection, WithHeadingRow, WithValidation, SkipsEmptyRows, WithChunkReading
class TransactionsSheet implements ToModel, WithHeadingRow, WithValidation, WithBatchInserts, WithUpserts, SkipsEmptyRows, WithEvents
{
use ValidatesPortfolioPermissions;
public function collection(Collection $transactions)
public function __construct(
public BackupImport $backupImport
) { }
/**
* @return array
*/
public function registerEvents(): array
{
$this->validatePortfolioPermissions($transactions);
Transaction::withoutEvents(function () use ($transactions) {
foreach ($transactions->sortBy('date') as $transaction) {
Transaction::where('id', $transaction['transaction_id'])
->firstOr(function () use ($transaction) {
$transaction = Transaction::make()->forceFill([
'id' => $transaction['transaction_id'],
'symbol' => $transaction['symbol'],
'portfolio_id' => $transaction['portfolio_id'],
'transaction_type' => $transaction['transaction_type'],
'quantity' => $transaction['quantity'],
'cost_basis' => $transaction['cost_basis'] ?? 0,
'sale_price' => $transaction['sale_price'],
'split' => $transaction['split'] ?? null,
'reinvested_dividend' => $transaction['reinvested_dividend'] ?? null,
'date' => $transaction['date'],
]);
$transaction->save();
return $transaction;
})
->syncToHolding();
return [
BeforeSheet::class => function(BeforeSheet $event) {
DB::commit();
$this->backupImport->update([
'message' => __('Importing transactions...'),
]);
DB::beginTransaction();
}
});
];
}
public function model(array $transaction)
{
$transaction = new Transaction([
'id' => $transaction['transaction_id'] ?? Str::uuid()->toString(),
'symbol' => strtoupper($transaction['symbol']),
'portfolio_id' => $transaction['portfolio_id'],
'transaction_type' => $transaction['transaction_type'],
'quantity' => $transaction['quantity'],
'cost_basis' => $transaction['cost_basis'] ?? 0,
'sale_price' => $transaction['sale_price'],
'split' => boolval($transaction['split']) ? 1 : 0,
'reinvested_dividend' => boolval($transaction['reinvested_dividend']) ? 1 : 0,
'date' => Carbon::parse($transaction['date'])->format('Y-m-d')
]);
// stub out related holding
Holding::firstOrCreate([
'symbol' => $transaction->symbol,
'portfolio_id' => $transaction->portfolio_id
], [
'quantity' => 0,
'average_cost_basis' => 0,
]);
return $transaction;
}
public function batchSize(): int
{
return 150;
}
public function uniqueBy()
{
return 'id';
}
public function rules(): array
@@ -54,7 +83,7 @@ class TransactionsSheet implements ToCollection, WithHeadingRow, WithValidation,
return [
'transaction_id' => ['sometimes', 'nullable'],
'symbol' => ['required', 'string'],
'portfolio_id' => ['required', 'exists:portfolios,id'],
'portfolio_id' => ['required', new PortfolioAccessValidationRule($this->backupImport->user_id)],
'quantity' => ['required', 'min:0', 'numeric'],
'transaction_type' => ['required', 'in:BUY,SELL'],
'date' => ['required', 'date'],
@@ -65,9 +94,4 @@ class TransactionsSheet implements ToCollection, WithHeadingRow, WithValidation,
'sale_price' => ['sometimes', 'nullable', 'min:0', 'numeric'],
];
}
public function chunkSize(): int
{
return 500;
}
}
app/Imports/ValidatesPortfolioPermissions.php
-25
View File
@@ -1,25 +0,0 @@
<?php
namespace App\Imports;
use Exception;
use App\Models\Portfolio;
trait ValidatesPortfolioPermissions {
public function validatePortfolioPermissions($collection)
{
$portfolios = auth()->user()->portfolios->pluck('id');
$collection->pluck('portfolio_id')->unique()->each(function($portfolio) use ($portfolios) {
if (
!$portfolios->contains($portfolio)
|| auth()->user()->cannot('fullAccess', Portfolio::find($portfolio))
) {
throw new Exception('You do not have permission to access that portfolio.');
}
});
}
}
app/Jobs/BackupImportJob.php
+64
View File
@@ -0,0 +1,64 @@
<?php
namespace App\Jobs;
use Throwable;
use App\Models\User;
use App\Models\BackupImport;
use Illuminate\Support\Facades\Auth;
use Maatwebsite\Excel\Facades\Excel;
use Illuminate\Foundation\Queue\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use App\Imports\BackupImport as BackupImportExcel;
class BackupImportJob implements ShouldQueue
{
use Queueable;
/**
* The number of times the job may be attempted.
*/
public $tries = 1;
/**
* The number of seconds the job can run before timing out.
*
* @var int
*/
public $timeout = 300;
/**
* Indicate if the job should be marked as failed on timeout.
*
* @var bool
*/
public $failOnTimeout = true;
/**
* Create a new job instance.
*/
public function __construct(
public BackupImport $backupImport
) { }
/**
* Execute the job.
*/
public function handle(): void
{
Excel::import(new BackupImportExcel($this->backupImport), $this->backupImport->path, config('livewire.temporary_file_upload.disk', null));
}
/**
* Handle a job failure.
*/
public function failed(?Throwable $e): void
{
$this->backupImport->update([
'status' => 'failed',
'message' => 'Error: '. substr($e->getMessage(), 0, 220),
'has_errors' => true,
'completed_at' => now()
]);
}
}
app/Models/BackupImport.php
+52
View File
@@ -0,0 +1,52 @@
<?php
namespace App\Models;
use Maatwebsite\Excel\Facades\Excel;
use Illuminate\Database\Eloquent\Model;
use App\Imports\BackupImport as BackupImportExcel;
use App\Jobs\BackupImportJob;
use Illuminate\Database\Eloquent\Concerns\HasUuids;
class BackupImport extends Model
{
use HasUuids;
protected $table = 'backup_import_jobs';
protected $fillable = [
'user_id',
'path',
'status', // pending, in_progress, success, failed
'message', // Import starting, Import is in progress, Importing portfolios, Importing transactions, Importing daily changes, Import completed successfully
'has_errors',
'completed_at'
];
protected static function boot()
{
parent::boot();
static::creating(function ($import) {
$import->status = 'pending';
$import->message = __('Import starting...');
});
static::created(function ($import) {
BackupImportJob::dispatch($import);
});
}
protected $hidden = [];
protected $appends = [];
protected function casts(): array
{
return [
'completed_at' => 'datetime'
];
}
}
app/Models/Portfolio.php
+3 -3
View File
@@ -69,10 +69,10 @@ class Portfolio extends Model
});
}
public function scopeFullAccess()
public function scopeFullAccess($query, $user_id = null)
{
return $this->whereHas('users', function ($query) {
$query->where('user_id', auth()->user()->id)
return $query->whereHas('users', function ($query) use ($user_id) {
$query->where('user_id', $user_id ?? auth()->user()->id)
->where(function ($query) {
$query->where('full_access', true)
->orWhere('owner', true);
app/Rules/PortfolioAccessValidationRule.php
+27
View File
@@ -0,0 +1,27 @@
<?php
namespace App\Rules;
use App\Models\Portfolio;
use Illuminate\Contracts\Validation\ValidationRule;
class PortfolioAccessValidationRule implements ValidationRule
{
public function __construct(
public ?string $user_id
) { }
/**
* Validate the attribute.
*
* @param string $attribute
* @param mixed $value
* @param \Closure $fail
* @return void
*/
public function validate(string $attribute, mixed $value, \Closure $fail): void
{
if (!Portfolio::fullAccess($this->user_id)->where('id', $value)->count()) {
$fail(__('You do not have access to that portfolio.'));
}
}
}