diff --git a/src/Migration/Destination.php b/src/Migration/Destination.php index f3206533..7ffc50db 100644 --- a/src/Migration/Destination.php +++ b/src/Migration/Destination.php @@ -2,6 +2,8 @@ namespace Utopia\Migration; +use Utopia\CLI\Console; + abstract class Destination extends Target { /** @@ -34,6 +36,8 @@ public function run( string $rootResourceId = '', string $rootResourceType = '', ): void { + Console::log(json_encode(['stage' => 'destination#run'], JSON_PRETTY_PRINT)); + $this->source->run( $resources, function (array $resources) use ($callback) { diff --git a/src/Migration/Source.php b/src/Migration/Source.php index fb4a146a..bed232f9 100644 --- a/src/Migration/Source.php +++ b/src/Migration/Source.php @@ -2,6 +2,8 @@ namespace Utopia\Migration; +use Utopia\CLI\Console; + abstract class Source extends Target { protected static int $defaultBatchSize = 100; @@ -54,6 +56,8 @@ public function callback(array $resources): void */ public function run(array $resources, callable $callback, string $rootResourceId = '', string $rootResourceType = ''): void { + Console::log(json_encode(['stage' => 'source#run'], JSON_PRETTY_PRINT)); + $this->rootResourceId = $rootResourceId; $this->rootResourceType = $rootResourceType; @@ -63,11 +67,27 @@ public function run(array $resources, callable $callback, string $rootResourceId /** @var Resource $resource */ if (!in_array($resource->getName(), $resources)) { $resource->setStatus(Resource::STATUS_SKIPPED); + Console::log(json_encode([ + 'stage' => 'transferCallback#1', + 'resource' => [ + 'data' => $resource, + 'status' => 'skipped', + ] + ], JSON_PRETTY_PRINT)); } else { $prunedResources[] = $resource; + Console::log(json_encode([ + 'stage' => 'transferCallback#2', + 'prunedResource' => $resource + ], JSON_PRETTY_PRINT)); } } + Console::log(json_encode([ + 'stage' => 'transferCallback#3', + 'status' => '$callback($returnedResources)' + ], JSON_PRETTY_PRINT)); + $callback($returnedResources); $this->cache->addAll($prunedResources); }; @@ -99,6 +119,12 @@ public function exportResources(array $resources): void } } + Console::log(json_encode([ + 'stage' => 'source#exportResources', + 'groups' => $groups, + 'resources' => $resources, + ], JSON_PRETTY_PRINT)); + if (empty($groups)) { return; } diff --git a/src/Migration/Sources/CSV.php b/src/Migration/Sources/CSV.php index aee145d9..9b98c691 100644 --- a/src/Migration/Sources/CSV.php +++ b/src/Migration/Sources/CSV.php @@ -2,6 +2,7 @@ namespace Utopia\Migration\Sources; +use Utopia\CLI\Console; use Utopia\Database\Database as UtopiaDatabase; use Utopia\Migration\Exception; use Utopia\Migration\Resource; @@ -59,6 +60,11 @@ public function report(array $resources = []): array $report = []; if (! $this->device->exists($this->filePath)) { + Console::log(json_encode([ + 'stage' => 'csv#report', + 'status' => "exiting, file doesn't exist." . ", path: " . $this->filePath + ], JSON_PRETTY_PRINT)); + return $report; } @@ -70,6 +76,8 @@ public function report(array $resources = []): array $report[Resource::TYPE_DOCUMENT] = $rowCount; + Console::log(json_encode(['stage' => 'csv#report', 'report' => $report], JSON_PRETTY_PRINT)); + return $report; } @@ -83,11 +91,23 @@ protected function exportGroupAuth(int $batchSize, array $resources): void protected function exportGroupDatabases(int $batchSize, array $resources): void { + Console::log(json_encode([ + 'stage' => 'csv#exportGroupDatabases', + 'resources' => $resources, + 'hasDocumentInScope' => in_array(Resource::TYPE_DOCUMENT, $resources) + ], JSON_PRETTY_PRINT)); + try { if (\in_array(Resource::TYPE_DOCUMENT, $resources)) { $this->exportDocuments($batchSize); } } catch (\Throwable $e) { + Console::log(json_encode([ + 'stage' => 'csv#exportGroupDatabases', + 'status' => 'error', + 'trace' => $e->getTraceAsString(), + ], JSON_PRETTY_PRINT)); + $this->addError( new Exception( Resource::TYPE_DOCUMENT, @@ -116,6 +136,12 @@ private function exportDocuments(int $batchSize): void $database = new Database($databaseId, ''); $collection = new Collection($database, '', $collectionId); + Console::log(json_encode([ + 'stage' => 'csv#exportDocuments', + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + ], JSON_PRETTY_PRINT)); + while (true) { $queries = [$this->database->queryLimit($batchSize)]; if ($lastAttribute) { @@ -138,6 +164,11 @@ private function exportDocuments(int $batchSize): void $attributeTypes = []; $manyToManyKeys = []; + Console::log(json_encode([ + 'stage' => 'csv#exportDocuments', + 'attributes' => $attributes, + ], JSON_PRETTY_PRINT)); + foreach ($attributes as $attribute) { $key = $attribute['key']; @@ -215,12 +246,23 @@ private function exportDocuments(int $batchSize): void if (count($buffer) === $batchSize) { $this->callback($buffer); + + Console::log(json_encode([ + 'stage' => 'csv#withCSVStream#callback', + 'buffer' => $buffer, + ], JSON_PRETTY_PRINT)); + $buffer = []; } } if (! empty($buffer)) { $this->callback($buffer); + + Console::log(json_encode([ + 'stage' => 'csv#withCSVStream#callback#2', + 'buffer' => $buffer, + ], JSON_PRETTY_PRINT)); } }); } @@ -296,6 +338,12 @@ private function validateCSVHeaders(array $headers, array $attributeTypes): void $extraAttributes = array_diff($filteredHeaders, $expectedAttributes); $missingAttributes = array_diff($expectedAttributes, $filteredHeaders); + Console::log(json_encode([ + 'stage' => 'csv#validateCSVHeaders', + 'missingAttributes' => $missingAttributes, + 'extraAttributes' => $extraAttributes, + ], JSON_PRETTY_PRINT)); + if (! empty($missingAttributes) || ! empty($extraAttributes)) { $messages = []; diff --git a/src/Migration/Transfer.php b/src/Migration/Transfer.php index aa660db9..f65d89df 100644 --- a/src/Migration/Transfer.php +++ b/src/Migration/Transfer.php @@ -2,6 +2,8 @@ namespace Utopia\Migration; +use Utopia\CLI\Console; + class Transfer { public const GROUP_GENERAL = 'general'; @@ -238,6 +240,15 @@ public function run( $this->resources = $computedResources; + Console::log(json_encode([ + 'stage' => 'transfer#run', + 'resources' => $resources, + 'rootResourceId' => $rootResourceId, + 'rootResourceType' => $rootResourceType, + 'computedResources' => $computedResources + ], JSON_PRETTY_PRINT)); + + $this->destination->run( $computedResources, $callback,