From 8c0bf85c79d511a887e3d655ccc590196b8e5d4a Mon Sep 17 00:00:00 2001 From: Ruben van der Linde Date: Mon, 2 Nov 2026 18:29:00 +0100 Subject: [PATCH] refactor(archimate): remove private methods nothing calls PHPStan reported 24 private methods and 2 constants in the ArchiMate import, export and service classes and in AangebodenGebruikService as unused (they were baselined). Removing them left 33 more private methods, 5 never-read properties and 2 constants without a caller; those go too, repeated until PHPStan reports none. No caller changes; the baseline loses the 26 entries that matched the removed code. Kept on purpose: AangebodenGebruikService::checkOrganisationOwnership and getApplicationsOwnedByOrganisation (unused ownership checks, to be decided separately). Assisted-by: ClaudeCode:claude-opus-5-5 --- lib/Service/AangebodenGebruikService.php | 141 +- lib/Service/ArchiMateExportService.php | 51 - lib/Service/ArchiMateImportService.php | 1310 +---------- lib/Service/ArchiMateService.php | 2618 +++------------------- phpstan-baseline.neon | 130 -- 5 files changed, 322 insertions(+), 3928 deletions(-) diff --git a/lib/Service/AangebodenGebruikService.php b/lib/Service/AangebodenGebruikService.php index b3006f95c..b08f04cd9 100644 --- a/lib/Service/AangebodenGebruikService.php +++ b/lib/Service/AangebodenGebruikService.php @@ -1165,28 +1165,13 @@ private function getGebruiksConfiguration(): array { ); }//end getGebruiksConfiguration() - /** - * Get configuration for koppelingen objects (register ID and schema IDs) - * - * @return array Configuration with register_id and schemas array - * @throws Exception When configuration is not available - */ - private function getKoppelingenConfiguration(): array { - return $this->resolveVoorzieningenSchemaConfig( - schemaKey: 'koppeling_schema', - labelForLogs: 'koppelingen' - ); - }//end getKoppelingenConfiguration() - /** * Resolves a `{register_id, schemas: [schemaId]}` tuple for the supplied * voorzieningen-config schema key. * - * Extracted from {@see getGebruiksConfiguration()} + - * {@see getKoppelingenConfiguration()} as part of task 7.3 — the two - * methods were 95% identical (differing only on the schema key and the - * log label). Throws when either the register id or the requested schema - * id is missing on the voorzieningen config. + * Used by {@see getGebruiksConfiguration()}. Throws when either the + * register id or the requested schema id is missing on the voorzieningen + * config. * * @param string $schemaKey The voorzieningen-config schema key * (e.g. "gebruik_schema"). @@ -1238,64 +1223,6 @@ private function resolveVoorzieningenSchemaConfig(string $schemaKey, string $lab throw new Exception(ucfirst($labelForLogs) . ' configuration not found. Please configure the schemas in the admin panel.'); }//end resolveVoorzieningenSchemaConfig() - /** - * Get all objects for a specific schema using paginated search, optionally filtered by organization - * - * Uses ObjectService's buildSearchQuery() for proper query construction - * - * @param ObjectServiceInterface $objectService The OpenRegister object service - * @param string $registerId The register ID - * @param string $schemaId The schema ID - * @param array $options Query options (includes request parameters for buildSearchQuery) - * @param string|null $organisationFilter Optional organization UUID to filter by - * - * @return array Paginated result from searchObjectsPaginated - * - * @throws Exception When query fails - */ - private function getAllObjectsForSchema( - ObjectServiceInterface $objectService, - string $registerId, - string $schemaId, - array $options = [], - ?string $organisationFilter = null, - ): array { - // Use ObjectService's buildSearchQuery to properly handle request parameters. - $searchQuery = $objectService->buildSearchQuery($options); - - // Add schema and register filters. - $searchQuery['@self']['schema'] = $schemaId; - $searchQuery['@self']['register'] = $registerId; - - // Add organization filter if provided. - if ($organisationFilter !== null) { - $searchQuery['@self']['organisation'] = $organisationFilter; - } - - // Force database source for real-time data. - $searchQuery['_source'] = 'database'; - - $this->logger->debug( - 'Getting all objects for schema (paginated)', - [ - 'register' => $registerId, - 'schema' => $schemaId, - 'organisation_filter' => $organisationFilter, - 'query' => $searchQuery, - ] - ); - - // Execute search with RBAC and multitenancy disabled using paginated search. - $searchResult = $objectService->searchObjectsPaginated( - query: $searchQuery, - _rbac: false, - _multitenancy: false, - deleted: false - ); - - return $searchResult; - }//end getAllObjectsForSchema() - /** * Get all applications/modules owned by an organization * @@ -1400,68 +1327,6 @@ private function getApplicationsOwnedByOrganisation( }//end try }//end getApplicationsOwnedByOrganisation() - /** - * Get objects related to a specific UUID via the uses relationship using paginated search - * - * Uses ObjectService's buildSearchQuery() for proper query construction - * - * @param ObjectServiceInterface $objectService The OpenRegister object service - * @param string $registerId The register ID - * @param string $schemaId The schema ID - * @param string $relatedUuid The UUID to find relationships for - * @param array $options Query options (includes request parameters for buildSearchQuery) - * @param string|null $organisationFilter Optional organization UUID to filter by - * - * @return array Paginated result from searchObjectsPaginated - * - * @throws Exception When query fails - */ - private function getObjectsRelatedToUuid( - ObjectServiceInterface $objectService, - string $registerId, - string $schemaId, - string $relatedUuid, - array $options = [], - ?string $organisationFilter = null, - ): array { - // Use ObjectService's buildSearchQuery to properly handle request parameters. - $searchQuery = $objectService->buildSearchQuery($options); - - // Add schema and register filters. - $searchQuery['@self']['schema'] = $schemaId; - $searchQuery['@self']['register'] = $registerId; - - // Add organization filter if provided. - if ($organisationFilter !== null) { - $searchQuery['@self']['organisation'] = $organisationFilter; - } - - // Force database source for real-time data. - $searchQuery['_source'] = 'database'; - - $this->logger->debug( - 'Getting objects related to UUID (paginated)', - [ - 'register' => $registerId, - 'schema' => $schemaId, - 'related_uuid' => $relatedUuid, - 'organisation_filter' => $organisationFilter, - 'query' => $searchQuery, - ] - ); - - // Execute search using the uses parameter to find relationships with pagination. - $searchResult = $objectService->searchObjectsPaginated( - query: $searchQuery, - _rbac: false, - _multitenancy: false, - deleted: false, - uses: $relatedUuid - ); - - return $searchResult; - }//end getObjectsRelatedToUuid() - /** * Check if an organization owns a specific application/module * diff --git a/lib/Service/ArchiMateExportService.php b/lib/Service/ArchiMateExportService.php index f94410edb..3d292cc23 100644 --- a/lib/Service/ArchiMateExportService.php +++ b/lib/Service/ArchiMateExportService.php @@ -1919,57 +1919,6 @@ private function addModelMetadataToXml(\SimpleXMLElement $xml, array $modelMetad } }//end addModelMetadataToXml() - /** - * Optimized method to add data to XML node. - * - * @param \SimpleXMLElement $node The XML node. - * @param array $data The data array. - * - * @return void - */ - private function addDataToXmlNode(\SimpleXMLElement $node, array $data): void { - // Add attributes first. - if (isset($data['_attributes']) === true && is_array($data['_attributes']) === true) { - foreach ($data['_attributes'] as $attrName => $attrValue) { - $node->addAttribute($attrName, (string)$attrValue); - } - } - - // Add text content. - if (isset($data['_value']) === true) { - $node[0] = (string)$data['_value']; - } - - // Add child elements. - foreach ($data as $key => $value) { - if ($key === '_attributes' || $key === '_value' || is_int($key) === true) { - continue; - } - - if (is_array($value) === true) { - if (isset($value[0]) === true) { - // Array of items. - foreach ($value as $item) { - $child = $node->addChild($key); - if (is_array($item) === true) { - $this->addDataToXmlNode(node: $child, data: $item); - } else { - $child[0] = (string)$item; - } - } - } else { - // Single object. - $child = $node->addChild($key); - $this->addDataToXmlNode(node: $child, data: $value); - } - } else { - // Scalar value. - $child = $node->addChild($key); - $child[0] = (string)$value; - }//end if - }//end foreach - }//end addDataToXmlNode() - /** * Convert OpenRegister objects back to ArchiMate format * diff --git a/lib/Service/ArchiMateImportService.php b/lib/Service/ArchiMateImportService.php index 7f06a382c..00223c7a6 100644 --- a/lib/Service/ArchiMateImportService.php +++ b/lib/Service/ArchiMateImportService.php @@ -67,38 +67,6 @@ * @SuppressWarnings(PHPMD.CountInLoopExpression) */ class ArchiMateImportService { - /** - * Configuration keys for ArchiMate processing - */ - private const CONFIG_KEYS = [ - 'archimate_register_id' => 'archimate_register_id', - 'archimate_schema_id' => 'archimate_schema_id', - 'archimate_model_schema_id' => 'archimate_model_schema_id', - ]; - - /** - * Performance optimization settings - */ - private const PERFORMANCE_OPTIMIZATIONS = [ - 'disable_validation' => true, - 'disable_events' => true, - 'disable_rbac' => false, - // Keep RBAC for security. - 'use_multi' => true, - 'xml_parse_flags' => LIBXML_NOCDATA | LIBXML_NONET, - 'memory_cleanup' => true, - 'parallel_processing' => true, - 'batch_size' => 1000, - // Default batch size (will be adjusted intelligently). - 'parallel_batches' => 8, - // Process 8 batches concurrently. - 'max_batch_size_bytes' => 8388608, - // 8 MB - safe under MySQL's 16 MB limit. - 'min_batch_size' => 50, - // Minimum batch size for very large objects. - 'size_estimation_sample' => 10, - // Sample size for estimating object sizes. - ]; /** * NOTE: Default schema IDs removed - all schema IDs must be configured via AMEF settings. @@ -333,21 +301,6 @@ public function xmlToArray(\SimpleXMLElement $xml): array { return $result; }//end xmlToArray() - /** - * Check if an array is associative (has string keys). - * - * @param mixed $value The value to check. - * - * @return bool True if associative, false if indexed - */ - private function isAssoc(mixed $value): bool { - if (is_array($value) === false) { - return false; - } - - return array_keys($value) !== range(0, count($value) - 1); - }//end isAssoc() - /** * OPTIMIZED: Import ArchiMate XML file using OpenRegister-style performance optimization * @@ -396,8 +349,6 @@ public function importArchiMateFileFromPathOptimized(array $options = []): array // PERFORMANCE OPTIMIZATION: Clean up memory after XML parsing. $memoryCleanupTime = 0; - // PERFORMANCE_OPTIMIZATIONS['memory_cleanup'] is a class constant set - // to true, so this was never conditional. $memCleanupStart = microtime(true); $this->cleanupMemory(); $memoryCleanupTime = microtime(true) - $memCleanupStart; @@ -1717,275 +1668,6 @@ private function fixStandaardVersieUuids(int $registerId): void { }//end try }//end fixStandaardVersieUuids() - /** - * Save objects directly to ObjectService without custom batching - * Lets ObjectService handle all batching, throttling, and optimization internally - * - * @param array $objects Array of objects to save - * @param ObjectServiceInterface $objectService ObjectService instance - * @param int $registerId Register ID - * - * @return array Array of saved objects - */ - private function saveObjectsDirectToService(array $objects, ObjectServiceInterface $objectService, int $registerId): array { - try { - // GROUP BY SCHEMA: For magic mapping support, save objects schema by schema. - // This ensures each batch has a single schema so UnifiedObjectMapper can route to the correct table. - $schemaGroups = []; - foreach ($objects as $obj) { - $schemaId = $obj['@self']['schema'] ?? 'unknown'; - $schemaGroups[$schemaId][] = $obj; - } - - $this->logger->info( - 'ArchiMate import: Grouped objects by schema', - [ - 'schemaCount' => count($schemaGroups), - 'schemas' => array_map('count', $schemaGroups), - ] - ); - - // Save each schema group separately. - $allSaved = []; - $allUpdated = []; - $allUnchanged = []; - $allSkipped = []; - $allInvalid = []; - - foreach ($schemaGroups as $schemaId => $schemaObjects) { - if ($schemaId !== 'unknown') { - $schemaValue = $schemaId; - } else { - $schemaValue = null; - } - - $saveResult = $objectService->saveObjects( - objects: $schemaObjects, - register: $registerId, - schema: $schemaValue, - _rbac: true, - _multitenancy: true, - validation: true, - events: true - ); - - // Merge results. - $allSaved = array_merge($allSaved, $saveResult['saved'] ?? []); - $allUpdated = array_merge($allUpdated, $saveResult['updated'] ?? []); - $allUnchanged = array_merge($allUnchanged, $saveResult['unchanged'] ?? []); - $allSkipped = array_merge($allSkipped, $saveResult['skipped'] ?? []); - $allInvalid = array_merge($allInvalid, $saveResult['invalid'] ?? []); - - $this->logger->debug( - 'Schema group saved', - [ - 'schemaId' => $schemaId, - 'objectCount' => count($schemaObjects), - 'saved' => count($saveResult['saved'] ?? []), - 'updated' => count($saveResult['updated'] ?? []), - ] - ); - }//end foreach - - // Combine all results. - $saveResult = [ - 'saved' => $allSaved, - 'updated' => $allUpdated, - 'unchanged' => $allUnchanged, - 'skipped' => $allSkipped, - 'invalid' => $allInvalid, - ]; - - // Store result for statistics. - $this->lastSaveResult = $saveResult; - - // DEBUG: Log ObjectService save result. - $this->logger->info( - 'ObjectService save result DEBUG', - [ - 'total_objects_sent' => count($objects), - 'saved_count' => count($allSaved), - 'updated_count' => count($allUpdated), - 'unchanged_count' => count($allUnchanged), - 'skipped_count' => count($allSkipped), - 'invalid_count' => count($allInvalid), - ] - ); - - // Return combined saved and updated objects. - return array_merge($allSaved, $allUpdated, $allUnchanged); - } catch (\Exception $e) { - $this->logger->error( - 'Error in direct ObjectService save', - [ - 'error' => $e->getMessage(), - 'object_count' => count($objects), - ] - ); - throw $e; - }//end try - }//end saveObjectsDirectToService() - - /** - * Save objects in parallel batches for maximum performance (DEPRECATED) - * - * @param array $objects Array of objects to save - * @param ObjectServiceInterface $objectService ObjectService instance - * @param int $registerId Register ID - * - * @return array Array of saved objects - */ - private function saveObjectsInParallelBatches(array $objects, ObjectServiceInterface $objectService, int $registerId): array { - $batchSize = self::PERFORMANCE_OPTIMIZATIONS['batch_size']; - $parallelBatches = self::PERFORMANCE_OPTIMIZATIONS['parallel_batches']; - - // INTELLIGENT BATCH SIZING: Create size-aware batches instead of fixed-size chunks. - $chunks = $this->createIntelligentBatches(objects: $objects); - $totalChunks = count($chunks); - - // Batch processing initialized. - $allResults = []; - $processedChunks = 0; - - // Accumulate statistics from all chunks (using new format). - $aggregatedStats = [ - 'saved' => [], - 'updated' => [], - 'unchanged' => [], - 'invalid' => [], - ]; - - // Process chunks sequentially but with larger batch sizes for better performance. - foreach ($chunks as $chunkIndex => $chunk) { - $chunkInputCount = count($chunk); - - try { - // Disable RBAC for bulk import when the performance optimisation flag is set. - if (self::PERFORMANCE_OPTIMIZATIONS['disable_rbac'] === true) { - $_rbacValue = false; - } else { - $_rbacValue = true; - } - - $saveResult = $objectService->saveObjects( - objects: $chunk, - register: $registerId, - schema: null, - _rbac: $_rbacValue, - _multitenancy: true, - validation: !self::PERFORMANCE_OPTIMIZATIONS['disable_validation'], - events: !self::PERFORMANCE_OPTIMIZATIONS['disable_events'] - ); - - // Calculate totals received back from this chunk. - $chunkSaved = count($saveResult['saved'] ?? []); - $chunkUpdated = count($saveResult['updated'] ?? []); - $chunkUnchanged = count($saveResult['unchanged'] ?? []); - $chunkInvalid = count($saveResult['invalid'] ?? []); - $chunkTotalReceived = $chunkSaved + $chunkUpdated + $chunkUnchanged + $chunkInvalid; - - // Accumulate statistics from this chunk. - $aggregatedStats['saved'] = array_merge($aggregatedStats['saved'], $saveResult['saved'] ?? []); - $aggregatedStats['updated'] = array_merge($aggregatedStats['updated'], $saveResult['updated'] ?? []); - $aggregatedStats['unchanged'] = array_merge($aggregatedStats['unchanged'], $saveResult['unchanged'] ?? []); - $aggregatedStats['invalid'] = array_merge($aggregatedStats['invalid'], $saveResult['invalid'] ?? []); - - $savedObjects = array_merge( - $saveResult['saved'] ?? [], - $saveResult['updated'] ?? [] - ); - - $allResults = array_merge($allResults, $savedObjects); - - $processedChunks++; - } catch (\Exception $e) { - $this->logger->error( - 'Error processing chunk', - [ - 'chunk_index' => $chunkIndex, - 'error' => $e->getMessage(), - ] - ); - // Continue with other chunks. - }//end try - - // Memory cleanup between chunks. - // PERFORMANCE_OPTIMIZATIONS['memory_cleanup'] is a class constant set - // to true, so this was never conditional. Flip the constant and the - // compiler will point you back here. - $this->cleanupMemory(); - }//end foreach - - // Store the aggregated result for statistics calculation. - $this->lastSaveResult = $aggregatedStats; - - $totalSaved = count($aggregatedStats['saved']); - $totalUpdated = count($aggregatedStats['updated']); - $totalUnchanged = count($aggregatedStats['unchanged']); - $totalInvalid = count($aggregatedStats['invalid']); - $totalObjProcessed = $totalSaved + $totalUpdated + $totalUnchanged + $totalInvalid; - - // Batch processing completed. - // Log critical discrepancy if found. - if (count($objects) !== $totalObjProcessed) { - $this->logger->critical( - 'OBJECT COUNT MISMATCH DETECTED', - [ - 'objects_sent_to_openregister' => count($objects), - 'objects_processed_by_openregister' => $totalObjProcessed, - 'missing_objects' => count($objects) - $totalObjProcessed, - 'this_explains_the_781_missing_objects' => true, - ] - ); - } - - return $allResults; - }//end saveObjectsInParallelBatches() - - /** - * Save objects in a single batch (fallback method) - * - * @param array $objects Array of objects to save - * @param ObjectServiceInterface $objectService ObjectService instance - * @param int $registerId Register ID - * - * @return array Array of saved objects - */ - private function saveObjectsInSingleBatch(array $objects, ObjectServiceInterface $objectService, int $registerId): array { - // Using single batch processing. - // Disable RBAC for bulk import when the performance optimisation flag is set. - if (self::PERFORMANCE_OPTIMIZATIONS['disable_rbac'] === true) { - $_rbacValue = false; - } else { - $_rbacValue = true; - } - - $saveResult = $objectService->saveObjects( - objects: $objects, - register: $registerId, - schema: null, - _rbac: $_rbacValue, - _multitenancy: true, - validation: !self::PERFORMANCE_OPTIMIZATIONS['disable_validation'], - events: !self::PERFORMANCE_OPTIMIZATIONS['disable_events'] - ); - - // Store the save result for later access to statistics. - $this->lastSaveResult = $saveResult; - - // Extract saved objects from the new structured return format. - $savedObjects = array_merge( - $saveResult['saved'] ?? [], - $saveResult['updated'] ?? [] - ); - - // Objects saved successfully. - // Validation errors logged if any. - // Unchanged objects noted if any. - // Return the combined saved and updated objects (maintaining backward compatibility). - return $savedObjects; - }//end saveObjectsInSingleBatch() - /** * Get ObjectService from container * @@ -2037,29 +1719,6 @@ private function initializeCache(): void { ]; }//end initializeCache() - /** - * Log current memory usage for performance monitoring - * - * @param string $stage Description of the current processing stage - * - * @return void - */ - private function logMemoryUsage(string $stage): void { - // Check if debug logging is available (Nextcloud logger doesn't have isDebug method). - $memoryUsage = memory_get_usage(true); - $memoryPeak = memory_get_peak_usage(true); - $memoryLimit = ini_get('memory_limit'); - - $this->logger->debug( - "Memory usage at: {$stage}", - [ - 'current_mb' => round($memoryUsage / 1024 / 1024, 2), - 'peak_mb' => round($memoryPeak / 1024 / 1024, 2), - 'limit' => $memoryLimit, - ] - ); - }//end logMemoryUsage() - /** * Clean up memory by forcing garbage collection * @@ -2559,84 +2218,6 @@ private function buildStatisticsFromSaveResult(): array { return $statistics; }//end buildStatisticsFromSaveResult() - /** - * Calculate optimized statistics for performance reporting - * - * Reads the persisted save outcome from {@see self::$lastSaveResult}. - * - * @return array Statistics array - */ - private function calculateOptimizedStatistics(): array { - // Initialize statistics structure for detailed error extraction. - $statistics = [ - 'elements' => ['created' => 0, 'updated' => 0, 'unchanged' => 0, 'errors' => []], - 'organizations' => ['created' => 0, 'updated' => 0, 'unchanged' => 0, 'errors' => []], - 'relationships' => ['created' => 0, 'updated' => 0, 'unchanged' => 0, 'errors' => []], - 'views' => ['created' => 0, 'updated' => 0, 'unchanged' => 0, 'errors' => []], - 'property_definitions' => ['created' => 0, 'updated' => 0, 'unchanged' => 0, 'errors' => []], - 'omschrijving' => [ - 'total_objects_created' => 0, - 'total_objects_updated' => 0, - 'total_objects_deleted' => 0, - 'total_objects_unchanged' => 0, - 'total_errors' => 0, - ], - ]; - - if ($this->lastSaveResult !== null) { - $saveResult = $this->lastSaveResult; - - // Use per-schema counts if available (reliable — doesn't depend on serialized object fields). - if (empty($saveResult['countsBySchema']) === false && $this->cachedConfig !== null) { - $sectionMap = [ - 'model' => 'elements', - 'element' => 'elements', - 'relationship' => 'relationships', - 'organization' => 'organizations', - 'view' => 'views', - 'property_definition' => 'property_definitions', - ]; - - $schemaToSection = []; - foreach ($this->cachedConfig['schemaIds'] as $type => $schemaId) { - $schemaToSection[(int)$schemaId] = $sectionMap[$type] ?? 'elements'; - } - - foreach ($saveResult['countsBySchema'] as $schemaId => $counts) { - $sectionKey = $schemaToSection[(int)$schemaId] ?? 'elements'; - $statistics[$sectionKey]['created'] += $counts['saved'] ?? 0; - $statistics[$sectionKey]['updated'] += $counts['updated'] ?? 0; - $statistics[$sectionKey]['unchanged'] += $counts['unchanged'] ?? 0; - if (($counts['invalid'] ?? 0) > 0) { - $statistics[$sectionKey]['errors'][] = "{$counts['invalid']} validation error(s)"; - } - } - }//end if - - // Calculate summary totals. - $summary = [ - 'total_objects_created' => 0, - 'total_objects_updated' => 0, - 'total_objects_deleted' => 0, - 'total_objects_unchanged' => 0, - 'total_errors' => 0, - ]; - - foreach ($statistics as $section => $sectionStats) { - if ($section !== 'omschrijving') { - $summary['total_objects_created'] += $sectionStats['created']; - $summary['total_objects_updated'] += $sectionStats['updated']; - $summary['total_objects_unchanged'] += $sectionStats['unchanged']; - $summary['total_errors'] += count($sectionStats['errors']); - } - } - - $statistics['omschrijving'] = $summary; - }//end if - - return $statistics; - }//end calculateOptimizedStatistics() - /** * Get section structure configuration for XML parsing * @@ -3223,246 +2804,61 @@ private function extractViewNodesRecursively($nodeData, array $elementsLookup = }//end extractViewNodesRecursively() /** - * Debug helper: Log parent-child relationships in view nodes - * - * @param array $viewNodes Array of view nodes with parent references - * @param string $viewId View identifier for logging context - * - * @return void - */ - private function debugViewNodeHierarchy(array $viewNodes, string $viewId): void { - $rootNodes = array_filter($viewNodes, fn ($node) => $node['parent'] === null); - $childNodes = array_filter($viewNodes, fn ($node) => $node['parent'] !== null); - - $this->logger->debug( - "View node hierarchy for view: {$viewId}", - [ - 'total_nodes' => count($viewNodes), - 'root_nodes' => count($rootNodes), - 'child_nodes' => count($childNodes), - 'parent_child_pairs' => array_map( - fn ($node) => ['child' => $node['viewNodeId'], 'parent' => $node['parent']], - $childNodes - ), - ] - ); - }//end debugViewNodeHierarchy() - - /** - * Recursively extract nested nodes with full hierarchy and element splicing (LEGACY - for backward compatibility) - * - * This method processes nodes and their children recursively to capture the complete - * nested structure as it appears in the ArchiMate XML. When a node references an element - * via elementRef, the actual element data (minus _xml) is spliced into the node's - * 'element' property. + * Extract all properties from an element for view node enrichment * - * @param array $nodeData Node data (can be single node or array of nodes) - * @param array $elementsLookup Lookup array of elements by identifier for splicing + * @param array $element Element data containing properties * - * @return array Array of processed nodes with nested children and spliced elements + * @return array Clean array of element properties */ - private function extractNodesRecursively($nodeData, array $elementsLookup = []): array { - $nodes = []; + private function extractElementProperties(array $element): array { + $properties = []; - // Handle both single node and array of nodes. - if (isset($nodeData[0]) === false) { - // Single node. - $nodeData = [$nodeData]; + // Extract basic element properties. + $basicProperties = ['identifier', 'name', 'omschrijving', 'section', 'model_identifier']; + foreach ($basicProperties as $prop) { + if (isset($element[$prop]) === true) { + $properties[$prop] = $element[$prop]; + } } - foreach ($nodeData as $node) { - if (isset($node['_attributes']) === true) { - $xValue = null; - $yValue = null; - $wValue = null; - $hValue = null; - if (isset($node['_attributes']['x']) === true) { - $xValue = (int)$node['_attributes']['x']; + // Extract all flattened properties (camelCase converted properties). + $excludedKeys = ['@self', 'xml', '_propertyMapping', '_slug', '_essential_data']; + foreach ($element as $key => $value) { + if (in_array($key, $excludedKeys) === false && in_array($key, $basicProperties) === false) { + // Only include non-object values or simple arrays. + $isSimple = is_scalar($value) === true + || (is_array($value) === true + && $this->isComplexArray(array: $value) === false); + if ($isSimple === true) { + $properties[$key] = $value; } + } + } - if (isset($node['_attributes']['y']) === true) { - $yValue = (int)$node['_attributes']['y']; - } + // Add property mapping for reference if available. + if (isset($element['_propertyMapping']) === true) { + $properties['_propertyMapping'] = $element['_propertyMapping']; + } - if (isset($node['_attributes']['w']) === true) { - $wValue = (int)$node['_attributes']['w']; - } + return $properties; + }//end extractElementProperties() - if (isset($node['_attributes']['h']) === true) { - $hValue = (int)$node['_attributes']['h']; - } + /** + * Check if an array contains complex nested structures + * + * @param array $array Array to check + * + * @return bool True if array contains complex nested structures + */ + private function isComplexArray(array $array): bool { + foreach ($array as $value) { + if (is_array($value) === true || is_object($value) === true) { + return true; + } + } - $processedNode = [ - 'identifier' => $node['_attributes']['identifier'] ?? null, - 'elementRef' => $node['_attributes']['elementRef'] ?? null, - 'type' => $node['_attributes']['xsi:type'] ?? 'Element', - 'x' => $xValue, - 'y' => $yValue, - 'w' => $wValue, - 'h' => $hValue, - ]; - - // Extract style information if present. - if (isset($node['style']) === true) { - $processedNode['style'] = $this->extractNodeStyle(style: $node['style']); - } - - // Extract label text for Label type nodes. - if (isset($node['label']) === true) { - if (is_array($node['label']) === true && isset($node['label']['_value']) === true) { - $processedNode['label'] = $node['label']['_value']; - } elseif (is_string($node['label']) === true) { - $processedNode['label'] = $node['label']; - } - } - - // ELEMENT SPLICING: If node references an element, splice it in. - if (empty($processedNode['elementRef']) === false && empty($elementsLookup) === false) { - $elementRef = $processedNode['elementRef']; - if (isset($elementsLookup[$elementRef]) === true) { - // Splice element data (minus _xml and other metadata) into the node. - $element = $elementsLookup[$elementRef]; - $processedNode['element'] = $this->prepareElementForSplicing(element: $element); - } - } - - // RECURSIVE: Extract child nodes if they exist (with element splicing). - if (isset($node['node']) === true) { - $processedNode['children'] = $this->extractNodesRecursively( - nodeData: $node['node'], - elementsLookup: $elementsLookup - ); - } - - // RECURSIVE: Extract child connections if they exist. - if (isset($node['connection']) === true) { - $processedNode['connections'] = $this->extractConnectionsRecursively( - connectionData: $node['connection'] - ); - } - - $nodes[] = $processedNode; - }//end if - }//end foreach - - return $nodes; - }//end extractNodesRecursively() - - /** - * Prepare element data for splicing by removing internal metadata - * - * @param array $element The complete element object - * - * @return array Element data suitable for splicing (without _xml, @self, etc.) - */ - private function prepareElementForSplicing(array $element): array { - // Start with a copy of the element. - $splicedElement = $element; - - // Remove internal metadata fields that shouldn't be in spliced data. - $fieldsToRemove = ['@self', 'xml', '_xml', 'section', 'model_identifier', 'extracted_at', '_propertyMapping']; - - foreach ($fieldsToRemove as $field) { - unset($splicedElement[$field]); - } - - return $splicedElement; - }//end prepareElementForSplicing() - - /** - * Extract connections recursively - * - * @param array $connectionData Connection data (can be single connection or array) - * - * @return array Array of processed connections - */ - private function extractConnectionsRecursively($connectionData): array { - $connections = []; - - // Handle both single connection and array of connections. - if (isset($connectionData[0]) === false) { - // Single connection. - $connectionData = [$connectionData]; - } - - foreach ($connectionData as $connection) { - if (isset($connection['_attributes']) === true) { - $processedConnection = [ - 'identifier' => $connection['_attributes']['identifier'] ?? null, - 'relationshipRef' => $connection['_attributes']['relationshipRef'] ?? null, - 'type' => $connection['_attributes']['xsi:type'] ?? 'Relationship', - 'source' => $connection['_attributes']['source'] ?? null, - 'target' => $connection['_attributes']['target'] ?? null, - ]; - - // Extract style information if present. - if (isset($connection['style']) === true) { - $processedConnection['style'] = $this->extractConnectionStyle(style: $connection['style']); - } - - $connections[] = $processedConnection; - } - } - - return $connections; - }//end extractConnectionsRecursively() - - /** - * Extract all properties from an element for view node enrichment - * - * @param array $element Element data containing properties - * - * @return array Clean array of element properties - */ - private function extractElementProperties(array $element): array { - $properties = []; - - // Extract basic element properties. - $basicProperties = ['identifier', 'name', 'omschrijving', 'section', 'model_identifier']; - foreach ($basicProperties as $prop) { - if (isset($element[$prop]) === true) { - $properties[$prop] = $element[$prop]; - } - } - - // Extract all flattened properties (camelCase converted properties). - $excludedKeys = ['@self', 'xml', '_propertyMapping', '_slug', '_essential_data']; - foreach ($element as $key => $value) { - if (in_array($key, $excludedKeys) === false && in_array($key, $basicProperties) === false) { - // Only include non-object values or simple arrays. - $isSimple = is_scalar($value) === true - || (is_array($value) === true - && $this->isComplexArray(array: $value) === false); - if ($isSimple === true) { - $properties[$key] = $value; - } - } - } - - // Add property mapping for reference if available. - if (isset($element['_propertyMapping']) === true) { - $properties['_propertyMapping'] = $element['_propertyMapping']; - } - - return $properties; - }//end extractElementProperties() - - /** - * Check if an array contains complex nested structures - * - * @param array $array Array to check - * - * @return bool True if array contains complex nested structures - */ - private function isComplexArray(array $array): bool { - foreach ($array as $value) { - if (is_array($value) === true || is_object($value) === true) { - return true; - } - } - - return false; - }//end isComplexArray() + return false; + }//end isComplexArray() /** * Apply style information to a viewNode structure @@ -3752,116 +3148,6 @@ private function extractConnectionType(array $connection): string { return 'association'; }//end extractConnectionType() - /** - * Extract style information from a node (LEGACY - for backward compatibility) - * - * @param array $style Style data from XML - * - * @return array Processed style information - */ - private function extractNodeStyle(array $style): array { - $processedStyle = []; - - // Extract fillColor. - if (isset($style['fillColor']['_attributes']) === true) { - $fillColor = $style['fillColor']['_attributes']; - $processedStyle['fillColor'] = [ - 'r' => (int)($fillColor['r'] ?? 255), - 'g' => (int)($fillColor['g'] ?? 255), - 'b' => (int)($fillColor['b'] ?? 255), - 'a' => (int)($fillColor['a'] ?? 100), - ]; - }//end if - - // Extract lineColor. - if (isset($style['lineColor']['_attributes']) === true) { - $lineColor = $style['lineColor']['_attributes']; - $processedStyle['lineColor'] = [ - 'r' => (int)($lineColor['r'] ?? 0), - 'g' => (int)($lineColor['g'] ?? 0), - 'b' => (int)($lineColor['b'] ?? 0), - 'a' => (int)($lineColor['a'] ?? 100), - ]; - }//end if - - // Extract font information. - if (isset($style['font']) === true) { - $font = []; - if (isset($style['font']['_attributes']) === true) { - $font['name'] = $style['font']['_attributes']['name'] ?? 'Arial'; - if (isset($style['font']['_attributes']['size']) === true) { - $font['size'] = (int)$style['font']['_attributes']['size']; - } else { - $font['size'] = 12; - } - } - - if (isset($style['font']['color']['_attributes']) === true) { - $fontColor = $style['font']['color']['_attributes']; - $font['color'] = [ - 'r' => (int)($fontColor['r'] ?? 0), - 'g' => (int)($fontColor['g'] ?? 0), - 'b' => (int)($fontColor['b'] ?? 0), - ]; - }//end if - - if (empty($font) === false) { - $processedStyle['font'] = $font; - } - }//end if - - return $processedStyle; - }//end extractNodeStyle() - - /** - * Extract style information from a connection - * - * @param array $style Style data from XML - * - * @return array Processed style information - */ - private function extractConnectionStyle(array $style): array { - $processedStyle = []; - - // Extract lineColor. - if (isset($style['lineColor']['_attributes']) === true) { - $lineColor = $style['lineColor']['_attributes']; - $processedStyle['lineColor'] = [ - 'r' => (int)($lineColor['r'] ?? 0), - 'g' => (int)($lineColor['g'] ?? 0), - 'b' => (int)($lineColor['b'] ?? 0), - ]; - }//end if - - // Extract font information. - if (isset($style['font']) === true) { - $font = []; - if (isset($style['font']['_attributes']) === true) { - $font['name'] = $style['font']['_attributes']['name'] ?? 'Arial'; - if (isset($style['font']['_attributes']['size']) === true) { - $font['size'] = (int)$style['font']['_attributes']['size']; - } else { - $font['size'] = 12; - } - } - - if (isset($style['font']['color']['_attributes']) === true) { - $fontColor = $style['font']['color']['_attributes']; - $font['color'] = [ - 'r' => (int)($fontColor['r'] ?? 0), - 'g' => (int)($fontColor['g'] ?? 0), - 'b' => (int)($fontColor['b'] ?? 0), - ]; - }//end if - - if (empty($font) === false) { - $processedStyle['font'] = $font; - } - }//end if - - return $processedStyle; - }//end extractConnectionStyle() - /** * Extract GEMMA type from an object using multiple possible property names * @@ -4458,133 +3744,6 @@ private function transformArchiMateXmlToObjectsBatch(array $xmlData, string $mod return $allObjects; }//end transformArchiMateXmlToObjectsBatch() - /** - * Transform views with performance optimizations - * - * This method processes views with several optimizations: - * - Reduced memory allocations - * - Optimized element lookup caching - * - Streamlined recursive processing - * - * @param array $viewsData Views section data - * @param string $modelIdentifier Model identifier - * @param array $propDefMap Property definition map - * @param array $elementsLookup Elements lookup for splicing - * - * @return array Array of processed view objects - */ - private function transformViewsOptimized( - array $viewsData, - string $modelIdentifier, - array $propDefMap, - array $elementsLookup, - ): array { - $objects = []; - - // Find items in section (optimized version for views). - $items = $this->findItemsSimplified(sectionData: $viewsData, sectionType: 'view'); - - // OPTIMIZATION: Pre-filter elements to only those actually referenced in views. - $referencedElements = $this->extractReferencedElements(viewItems: $items); - $filteredLookup = array_intersect_key($elementsLookup, array_flip($referencedElements)); - - $this->logger->debug( - 'Optimized elements lookup for views', - [ - 'total_elements' => count($elementsLookup), - 'referenced_elements' => count($filteredLookup), - 'optimization_ratio' => round( - (1 - count($filteredLookup) / max(count($elementsLookup), 1)) * 100, - 1 - ) . '%', - ] - ); - - foreach ($items as $item) { - if (is_array($item) === false) { - continue; - } - - $identifier = $this->extractIdentifier(item: $item, sectionName: 'view'); - if ($identifier === null) { - continue; - } - - // OPTIMIZATION: Use filtered elements lookup for better performance. - $essentialXmlData = $this->extractEssentialXmlData( - item: $item, - elementsLookup: $filteredLookup, - schemaType: 'view' - ); - - $object = [ - '@self' => [ - 'register' => $this->cachedConfig['registerId'] ?? throw new \RuntimeException('No register ID.'), - 'schema' => $this->getSchemaIdForSection(section: 'view'), - 'id' => $identifier, - 'owner' => $this->cachedConfig['userId'], - 'organisation' => $this->getCurrentOrganisation(), - ], - 'identifier' => $identifier, - 'section' => 'view', - 'model_identifier' => $modelIdentifier, - 'xml' => $essentialXmlData, - ]; - - // Extract name and summary (same as other sections). - if (isset($item['name']) === true) { - if (is_array($item['name']) === true && isset($item['name']['_value']) === true) { - $object['name'] = $item['name']['_value']; - } elseif (is_string($item['name']) === true) { - $object['name'] = $item['name']; - } - } - - if (isset($item['documentation']) === true) { - if (is_array($item['documentation']) === true && isset($item['documentation']['_value']) === true) { - $object['omschrijving'] = $item['documentation']['_value']; - } elseif (is_string($item['documentation']) === true) { - $object['omschrijving'] = $item['documentation']; - } - } - - // Extract type from xsi:type attribute. - if (isset($item['_xsi__type']) === true) { - $object['type'] = $item['_xsi__type']; - } elseif (isset($item['_attributes']['xsi:type']) === true) { - $object['type'] = $item['_attributes']['xsi:type']; - } - - // Flatten properties efficiently (same as other sections). - if (isset($item['properties']['property']) === true && empty($propDefMap) === false) { - $this->flattenPropertiesBatch( - object: $object, - properties: $item['properties']['property'], - propDefMap: $propDefMap - ); - - // Keep @self.id as the full ArchiMate identifier (set above). - // so stored IDs match GEMMA Online URLs (id-e0f57689-...). - $object['@self']['slug'] = $identifier; - } else { - $object['@self']['slug'] = $identifier; - } - - // Copy viewNodes and viewRelationships from XML to root level for easy access. - if (isset($object['xml']['viewNodes']) === true) { - $object['viewNodes'] = $object['xml']['viewNodes']; - } - - if (isset($object['xml']['viewRelationships']) === true) { - $object['viewRelationships'] = $object['xml']['viewRelationships']; - } - - $objects[] = $object; - }//end foreach - - return $objects; - }//end transformViewsOptimized() - /** * Extract all element references from view items for optimization * @@ -4852,217 +4011,6 @@ private function findSectionData(array $xmlData, string $sectionName): array { return []; }//end findSectionData() - /** - * Transform section objects in batch with minimal overhead and element splicing for views - * - * @param array $sectionData Section data from XML - * @param string $schemaType Schema type (singular) - * @param string $modelIdentifier Model identifier - * @param array $propDefMap Property definition map - * @param array $elementsLookup Optional elements lookup for view processing - * - * @return array Array of transformed objects - */ - private function transformSectionObjectsBatch( - array $sectionData, - string $schemaType, - string $modelIdentifier, - array $propDefMap, - array $elementsLookup = [], - ): array { - $objects = []; - - // Find items in section (simplified version). - $items = $this->findItemsSimplified(sectionData: $sectionData, sectionType: $schemaType); - - $skippedNotArray = 0; - $skippedNoIdentifier = 0; - - foreach ($items as $item) { - if (is_array($item) === false) { - $skippedNotArray++; - continue; - } - - $identifier = $this->extractIdentifier(item: $item, sectionName: $schemaType); - if ($identifier === null) { - $skippedNoIdentifier++; - continue; - } - - // Create object directly (minimal processing) with element splicing for views. - $essentialXmlData = $this->extractEssentialXmlData( - item: $item, - elementsLookup: $elementsLookup, - schemaType: $schemaType - ); - - $regId = $this->cachedConfig['registerId'] ?? throw new \RuntimeException('No register ID.'); - $sId = $this->cachedConfig['schemaIds'][$schemaType] ?? throw new \RuntimeException('No schema.'); - $object = [ - '@self' => [ - 'register' => $regId, - 'schema' => $sId, - 'id' => $identifier, - 'owner' => $this->cachedConfig['userId'], - 'organisation' => $this->getCurrentOrganisation(), - 'published' => date('Y-m-d\TH:i:s\Z'), - ], - 'identifier' => $identifier, - 'section' => $schemaType, - 'model_identifier' => $modelIdentifier, - 'xml' => $essentialXmlData, - ]; - - // Debug: Log XML data extraction. - $propsStructVal = null; - if (isset($item['properties']) === true) { - $propsStructVal = array_keys($item['properties']); - } - - $this->logger->debug( - 'XML data extracted for object', - [ - 'object_id' => $identifier, - 'section' => $schemaType, - 'original_item_keys' => array_keys($item), - 'essential_xml_keys' => array_keys($essentialXmlData), - 'essential_xml_size' => strlen(json_encode($essentialXmlData)), - 'has_properties' => isset($item['properties']) === true, - 'properties_structure' => $propsStructVal, - ] - ); - - // Extract name from XML if it exists. - if (isset($item['name']) === true) { - if (is_array($item['name']) === true && isset($item['name']['_value']) === true) { - $object['name'] = $item['name']['_value']; - } elseif (is_string($item['name']) === true) { - $object['name'] = $item['name']; - } - } - - // Extract documentation from XML if it exists and set to summary. - if (isset($item['documentation']) === true) { - if (is_array($item['documentation']) === true && isset($item['documentation']['_value']) === true) { - $object['omschrijving'] = $item['documentation']['_value']; - } elseif (is_string($item['documentation']) === true) { - $object['omschrijving'] = $item['documentation']; - } - } - - // Extract type from xsi:type attribute (e.g., "Capability", "ApplicationComponent"). - if (isset($item['_xsi__type']) === true) { - $object['type'] = $item['_xsi__type']; - } elseif (isset($item['_attributes']['xsi:type']) === true) { - $object['type'] = $item['_attributes']['xsi:type']; - } - - // Flatten properties efficiently (if present). - if (isset($item['properties']['property']) === true && empty($propDefMap) === false) { - $this->flattenPropertiesBatch( - object: $object, - properties: $item['properties']['property'], - propDefMap: $propDefMap - ); - - // FIXED: After properties are flattened, update ID and slug if objectId is available. - if (isset($object['objectId']) === true) { - // Use objectId as main ID and AMEF identifier as slug. - $object['@self']['id'] = $object['objectId']; - $object['@self']['slug'] = $identifier; - // AMEF identifier becomes slug. - } else { - // Fallback: extract clean UUID from AMEF identifier for slug. - if (str_starts_with($identifier, 'id-') === true) { - $object['@self']['slug'] = substr($identifier, 3); - // Remove "id-" prefix. - } else { - $object['@self']['slug'] = $identifier; - } - } - } else { - // No properties to flatten, use AMEF identifier logic. - if (str_starts_with($identifier, 'id-') === true) { - $object['@self']['slug'] = substr($identifier, 3); - // Remove "id-" prefix. - } else { - $object['@self']['slug'] = $identifier; - } - }//end if - - // NEW: For view objects, copy viewNodes and viewRelationships from XML to root level. - if ($schemaType === 'view' && isset($object['xml']) === true) { - if (isset($object['xml']['viewNodes']) === true) { - $object['viewNodes'] = $object['xml']['viewNodes']; - } - - if (isset($object['xml']['viewRelationships']) === true) { - $object['viewRelationships'] = $object['xml']['viewRelationships']; - } - } - - // DEBUG: Log final object structure before adding to array. - $xmlKeysValue = null; - if (isset($object['xml']) === true) { - $xmlKeysValue = array_keys($object['xml']); - } - - $propMapCountVal = 0; - if (isset($object['_propertyMapping']) === true) { - $propMapCountVal = count($object['_propertyMapping']); - } - - $viewNodesCountValue = 0; - if (isset($object['viewNodes']) === true) { - $viewNodesCountValue = count($object['viewNodes']); - } - - $viewRelCountVal = 0; - if (isset($object['viewRelationships']) === true) { - $viewRelCountVal = count($object['viewRelationships']); - } - - $this->logger->debug( - 'Final object structure before save', - [ - 'object_id' => $identifier, - 'section' => $schemaType, - 'object_keys' => array_keys($object), - 'has_xml_property' => isset($object['xml']) === true, - 'xml_keys' => $xmlKeysValue, - 'has_property_mapping' => isset($object['_propertyMapping']) === true, - 'property_mapping_count' => $propMapCountVal, - 'viewNodes_count' => $viewNodesCountValue, - 'viewRelationships_count' => $viewRelCountVal, - 'sample_properties' => array_slice( - array_diff( - array_keys($object), - [ - '@self', - 'identifier', - 'section', - 'model_identifier', - 'xml', - '_propertyMapping', - 'name', - 'omschrijving', - 'viewNodes', - 'viewRelationships', - ] - ), - 0, - 5 - ), - ] - ); - - $objects[] = $object; - }//end foreach - - return $objects; - }//end transformSectionObjectsBatch() - /** * Simplified item finding for better performance * @@ -5639,176 +4587,6 @@ private function bulkTransformViews( return $objects; }//end bulkTransformViews() - /** - * Create intelligent batches based on object size to prevent MySQL packet size issues - * - * This method analyzes object sizes and creates batches that stay under the MySQL - * max_allowed_packet limit while maintaining reasonable performance. - * - * TODO: Move this intelligent batch sizing to OpenRegister core as a native feature - * This functionality should be available for all bulk operations, not just ArchiMate imports. - * OpenRegister's saveObjects() method should handle this automatically based on object sizes. - * - * @param array $objects Array of objects to batch - * - * @return array Array of batches, each containing objects that fit within size limits - */ - private function createIntelligentBatches(array $objects): array { - $maxBatchSizeBytes = self::PERFORMANCE_OPTIMIZATIONS['max_batch_size_bytes']; - $minBatchSize = self::PERFORMANCE_OPTIMIZATIONS['min_batch_size']; - $sampleSize = self::PERFORMANCE_OPTIMIZATIONS['size_estimation_sample']; - - if (empty($objects) === true) { - return []; - } - - // Estimate average object size by sampling. - $avgObjectSize = $this->estimateAverageObjectSize(objects: $objects, sampleSize: $sampleSize); - - // Calculate optimal batch size based on object size. - $optimalBatchSize = max($minBatchSize, intval($maxBatchSizeBytes / $avgObjectSize)); - - $this->logger->info( - 'Intelligent batch sizing analysis', - [ - 'total_objects' => count($objects), - 'estimated_avg_object_size_bytes' => $avgObjectSize, - 'max_batch_size_bytes' => $maxBatchSizeBytes, - 'calculated_optimal_batch_size' => $optimalBatchSize, - 'min_batch_size_enforced' => $minBatchSize, - ] - ); - - // Create batches with size awareness. - $batches = []; - $currentBatch = []; - $currentBatchSize = 0; - - foreach ($objects as $object) { - $objectSize = $this->estimateObjectSize(object: $object); - - // Check if adding this object would exceed the batch size limit. - if (empty($currentBatch) === false && ($currentBatchSize + $objectSize) > $maxBatchSizeBytes) { - // Current batch is full, save it and start a new one. - $batches[] = $currentBatch; - $currentBatch = [$object]; - $currentBatchSize = $objectSize; - } else { - // Add object to current batch. - $currentBatch[] = $object; - $currentBatchSize += $objectSize; - } - - // Safety check: if a single object is larger than max batch size,. - // create a batch with just that object. - if (count($currentBatch) === 1 && $objectSize > $maxBatchSizeBytes) { - $this->logger->warning( - 'Very large object detected, creating single-object batch', - [ - 'object_id' => $object['@self']['id'] ?? 'unknown', - 'object_size_bytes' => $objectSize, - 'max_batch_size_bytes' => $maxBatchSizeBytes, - ] - ); - $batches[] = $currentBatch; - $currentBatch = []; - $currentBatchSize = 0; - } - }//end foreach - - // Add the last batch if it has objects. - if (empty($currentBatch) === false) { - $batches[] = $currentBatch; - } - - $this->logger->info( - 'Intelligent batching completed', - [ - 'total_objects' => count($objects), - 'total_batches_created' => count($batches), - 'batch_sizes' => array_map('count', $batches), - 'estimated_batch_sizes_bytes' => array_map( - fn ($batch) => array_sum(array_map([$this, 'estimateObjectSize'], $batch)), - $batches - ), - ] - ); - - return $batches; - }//end createIntelligentBatches() - - /** - * Estimate the average size of objects by sampling - * - * @param array $objects Array of objects to sample - * @param int $sampleSize Number of objects to sample for size estimation - * - * @return int Estimated average object size in bytes - */ - private function estimateAverageObjectSize(array $objects, int $sampleSize): int { - $totalObjects = count($objects); - if ($totalObjects === 0) { - return 1000; - // Default fallback size. - } - - // Sample evenly distributed objects. - $sampleIndices = []; - if ($totalObjects <= $sampleSize) { - // Use all objects if we have fewer than sample size. - $sampleIndices = range(0, $totalObjects - 1); - } else { - // Sample evenly across the array. - $step = max(1, intval($totalObjects / $sampleSize)); - for ($i = 0; $i < $totalObjects; $i += $step) { - $sampleIndices[] = $i; - if (count($sampleIndices) >= $sampleSize) { - break; - } - } - } - - // Calculate sizes of sampled objects. - $totalSampleSize = 0; - foreach ($sampleIndices as $index) { - $totalSampleSize += $this->estimateObjectSize(object: $objects[$index]); - } - - $averageSize = intval($totalSampleSize / count($sampleIndices)); - - $this->logger->debug( - 'Object size estimation completed', - [ - 'total_objects' => $totalObjects, - 'sampled_objects' => count($sampleIndices), - 'total_sample_size_bytes' => $totalSampleSize, - 'estimated_average_size_bytes' => $averageSize, - ] - ); - - return max(1000, $averageSize); - // Minimum 1KB per object. - }//end estimateAverageObjectSize() - - /** - * Estimate the serialized size of an object for batching purposes - * - * @param array $object The object to estimate size for - * - * @return int Estimated size in bytes - */ - private function estimateObjectSize(array $object): int { - // Quick estimation based on JSON serialization. - // This includes overhead for SQL parameters and structure. - $jsonSize = strlen(json_encode($object)); - - // Add overhead for SQL INSERT statement structure. - // Each object becomes multiple parameters in a bulk INSERT. - $sqlOverhead = 500; - // Estimated overhead per object in SQL. - return $jsonSize + $sqlOverhead; - }//end estimateObjectSize() - /** * Calculate detailed object statistics for import operations * diff --git a/lib/Service/ArchiMateService.php b/lib/Service/ArchiMateService.php index b65f1ff8a..e1258489c 100644 --- a/lib/Service/ArchiMateService.php +++ b/lib/Service/ArchiMateService.php @@ -82,76 +82,11 @@ class ArchiMateService { */ private array $camelCaseCache = []; - /** - * Cache for identifier extraction patterns by section type - * - * @var array - */ - private array $identifierPatternCache = []; - - /** - * Cache for property definition maps to avoid rebuilding during import - * - * @var array|null - */ - private ?array $propDefMapCache = null; - - /** - * Flag to track if we've already logged finding a GEMMA type property - * - * @var boolean - */ - private bool $gemmaTypePropertyFound = false; - private const CONFIG_KEYS = [ - 'archimate_register_id' => 'archimate_register_id', - 'archimate_schema_id' => 'archimate_schema_id', - 'archimate_model_schema_id' => 'archimate_model_schema_id', - ]; - - /** - * Performance optimization settings - */ - private const PERFORMANCE_OPTIMIZATIONS = [ - 'disable_validation' => true, - 'disable_events' => true, - 'disable_rbac' => false, - // Keep RBAC for security. - 'use_multi' => true, - 'xml_parse_flags' => LIBXML_NOCDATA | LIBXML_NONET, - 'memory_cleanup' => true, - 'parallel_processing' => true, - 'batch_size' => 1000, - // Default batch size (will be adjusted intelligently). - 'parallel_batches' => 8, - // Process 8 batches concurrently. - 'max_batch_size_bytes' => 8388608, - // 8 MB - safe under MySQL's 16 MB limit. - 'min_batch_size' => 50, - // Minimum batch size for very large objects. - 'size_estimation_sample' => 10, - // Sample size for estimating object sizes. - ]; - /** * NOTE: Default schema IDs removed - all schema IDs must be configured via AMEF settings. * The system will fail gracefully with clear error messages if configuration is missing. */ - /** - * Storage for the last save operation results. - * Contains the structured return from ObjectService::saveObjects. - * - * @var array|null - */ - private ?array $lastSaveResult = null; - - /** - * Cached configuration values for performance optimization. - * - * @var array|null - */ - private ?array $cachedConfig = null; - /** * Constructor for ArchiMateService * @@ -593,1157 +528,238 @@ private function createSchemaIdMap(): array { }//end createSchemaIdMap() /** - * Get section structure configuration for XML parsing - * - * @param string $sectionName The name of the section (e.g., 'elements', 'relationships', 'views', etc.) + * Get ObjectService from container * - * @return array Configuration with direct_tags and nested_paths for finding items + * @return ObjectServiceInterface|null ObjectService instance or null if not available */ - private function getSectionStructureConfig(string $sectionName): array { - // Define the structure configuration for each section type. - $configs = [ - 'elements' => [ - 'direct_tags' => ['element', 'elements'], - 'nested_paths' => [ - ['model', 'elements', 'element'], - ['model', 'elements'], - ['elements', 'element'], - ['elements'], - ], - ], - 'relationships' => [ - 'direct_tags' => ['relationship', 'relationships'], - 'nested_paths' => [ - ['model', 'relationships', 'relationship'], - ['model', 'relationships'], - ['relationships', 'relationship'], - ['relationships'], - ], - ], - 'views' => [ - 'direct_tags' => ['view', 'views', 'diagram', 'diagrams'], - 'nested_paths' => [ - ['model', 'views', 'diagrams', 'view'], - ['model', 'views', 'diagrams'], - ['model', 'views'], - ['views', 'diagrams', 'view'], - ['views', 'diagrams'], - ['views'], - ], - ], - 'organizations' => [ - 'direct_tags' => ['item', 'items'], - 'nested_paths' => [ - ['model', 'organizations', 'item'], - ['model', 'organizations'], - ['organizations', 'item'], - ['organizations'], - ], - ], - 'property_definitions' => [ - 'direct_tags' => ['propertyDefinition', 'propertyDefinitions'], - 'nested_paths' => [ - ['model', 'propertyDefinitions', 'propertyDefinition'], - ['model', 'propertyDefinitions'], - ['propertyDefinitions', 'propertyDefinition'], - ['propertyDefinitions'], - ], - ], - ]; + private function getObjectService(): ?ObjectServiceInterface { + if ($this->appManager->isInstalled(appId: 'openregister') === false) { + return null; + } - return $configs[$sectionName] ?? [ - 'direct_tags' => [$sectionName], - 'nested_paths' => [[$sectionName]], - ]; - }//end getSectionStructureConfig() + try { + return $this->container->get(ObjectService::class); + } catch (\Exception $e) { + $this->logger->warning( + 'Failed to get ObjectService', + [ + 'error' => $e->getMessage(), + ] + ); + return null; + } + }//end getObjectService() /** - * Check if an array is associative (has string keys) - * - * @param array $array The array to check - * - * @return bool True if associative, false if indexed + * NOTE: Removed deprecated methods getArchiMateRegisterId() and getArchiMateModelSchemaId() + * that had hardcoded fallbacks. All configuration is now retrieved via SettingsService + * through the initializeCache() method and proper AMEF configuration. */ - private function isAssociativeArray(array $array): bool { - return count(array_filter(array_keys($array), 'is_string')) > 0; - }//end isAssociativeArray() /** - * Find items within a specific section using AMEF configuration + * Read a configured id, failing closed on the empty default. + * + * The legacy fallback used to read every id with `''` as its default, so + * an unconfigured instance produced a config array full of empty STRINGS. + * Its consumers guard with `=== null` — `ViewService::getViews()` and + * `getView()` throw only when a value `=== null` — and `'' === null` is + * false. Today nothing reaches a query only because that fallback writes + * PLURAL key names (`views_schema`) while every consumer reads SINGULAR + * ones (`view_schema`), so the lookups miss and fall back to `null`. That + * is an accident of naming, not a defence: adding the singular keys — the + * obvious "cleanup" — would send `register => ''` straight into + * `searchObjects()` as an UNPINNED query, and an unpinned query returns + * rows, which reads exactly like a correct result. + * + * Returning null instead of `''` makes `?? null` downstream yield null, + * which is what every consumer already checks for, and the warning names + * the missing key so a misconfigured import stops reporting "0 objects" + * with no explanation. * - * @param array $sectionData The section data to search - * @param string $sectionName The name of the section + * @param string $key The app-config key holding the id. + * + * @return string|null The configured id, or null when it is unset. * - * @return array Array of items found + * @spec openspec/specs/archimate-import/spec.md */ - private function findItemsInSection(array $sectionData, string $sectionName): array { - // OPTIMIZATION: Removed debug logging from section processing. - $items = []; - - // No is_array() safety check: $sectionData is declared array, so PHP - // rejects anything else at the call boundary before this could run. - - // Get section structure configuration from AMEF config. - $config = $this->getSectionStructureConfig(sectionName: $sectionName); - - // Special handling for views with diagrams structure. - if ($sectionName === 'views') { - // Handle nested structure: . - if (isset($sectionData['diagrams']) === true) { - if (isset($sectionData['diagrams']['view']) === true) { - $viewArray = $sectionData['diagrams']['view']; - - // Handle single view vs array of views. - if (isset($viewArray[0]) === false && isset($viewArray['_attributes']) === true) { - // Single view. - $items = [$viewArray]; - } else { - // Array of views. - $items = $viewArray; - } - } - } else { - // Direct views structure (fallback). - if (isset($sectionData['view']) === true) { - $items = $sectionData['view']; - } - } - } else { - // Try to find items using the configured paths for other sections. - foreach ($config['nested_paths'] as $path) { - $currentData = $sectionData; - $pathValid = true; - - foreach ($path as $key) { - if (isset($currentData[$key]) === true) { - $currentData = $currentData[$key]; - } else { - $pathValid = false; - break; - } - } + private function resolveConfiguredId(string $key): ?string { + $value = $this->config->getValueString('stackiq', $key, ''); + if (trim($value) === '') { + $this->logger->warning( + 'ArchiMate configuration is incomplete — this id is not configured, so it is omitted rather than passed on as an empty string', + ['key' => $key] + ); - if ($pathValid !== false && is_array($currentData) === true) { - // Check if this is a direct array of items or needs further processing. - if (isset($currentData[0]) === true || $this->isAssociativeArray(array: $currentData) === true) { - $items = $currentData; - break; - } - } - }//end foreach - }//end if - - // If no items found through nested paths, try direct tags. - if (empty($items) === true) { - foreach ($config['direct_tags'] as $tag) { - if (isset($sectionData[$tag]) === true) { - $items = $sectionData[$tag]; - break; - } - } + return null; } - // If still no items found, treat the section itself as items. - if (empty($items) === true) { - $items = [$sectionData]; - } + return $value; + }//end resolveConfiguredId() - // Ensure items is always an array. - if (is_array($items) === false) { - $items = [$items]; - } + /** + * Get AMEF configuration from app config + * + * @return array AMEF configuration + * @spec openspec/specs/archimate-import/spec.md + */ + public function getAmefConfig(): array { + $this->logger->info('Getting AMEF configuration'); - // If items is an associative array with numeric keys, convert to indexed array. - if ($this->isAssociativeArray(array: $items) === true) { - $items = array_values($items); - } + try { + // Get configuration from app config using the correct method. + $config = $this->config->getValueString('stackiq', 'amef_config', '{}'); + $decoded = json_decode($config, true); + + if (is_array($decoded) === false) { + // Fallback to individual config values for backward + // compatibility. Every id is read through + // resolveConfiguredId(), which guards the empty default at the + // point of the read and omits the key entirely when it is + // unset — so `?? null` downstream yields null, which is what + // the consumers already check for. + $decoded = array_filter( + [ + 'register_id' => $this->resolveConfiguredId(key: 'amef_register'), + 'model_schema_id' => $this->resolveConfiguredId(key: 'amef_model_schema'), + 'elements_schema' => $this->resolveConfiguredId(key: 'amef_elements_schema'), + 'relationships_schema' => $this->resolveConfiguredId(key: 'amef_relationships_schema'), + 'views_schema' => $this->resolveConfiguredId(key: 'amef_views_schema'), + 'organizations_schema' => $this->resolveConfiguredId(key: 'amef_organizations_schema'), + 'folders_schema' => $this->resolveConfiguredId(key: 'amef_folders_schema'), + 'property_definitions_schema' => $this->resolveConfiguredId(key: 'amef_property_definitions_schema'), + ], + static function ($value) { + return $value !== null; + } + ); + }//end if + + return $decoded; + } catch (\Exception $e) { + $this->logger->error( + 'Failed to get AMEF configuration', + [ + 'error' => $e->getMessage(), + ] + ); - return $items; - }//end findItemsInSection() + return [ + 'success' => false, + 'error' => $e->getMessage(), + ]; + }//end try + }//end getAmefConfig() /** - * Extract identifier from item data - * - * @param array $item Item data - * @param string $sectionName The section name for special handling + * Get the current status of ArchiMate operations * - * @return string|null Identifier or null if not found + * @return array Status information including import/export status and object counts + * @spec openspec/specs/archimate-import/spec.md */ - private function extractIdentifier(array $item, string $sectionName = ''): ?string { - // OPTIMIZATION: Use cached patterns for section-specific identifier extraction. - if (isset($this->identifierPatternCache[$sectionName]) === true) { - $patterns = $this->identifierPatternCache[$sectionName]; - - // Try cached patterns in order of success frequency. - foreach ($patterns as $pattern) { - $result = $this->extractIdentifierByPattern(item: $item, pattern: $pattern); - if ($result !== null) { - return $result; - } + public function getArchiMateStatus(): array { + $this->logger->info('Getting ArchiMate status'); + + try { + // Get basic status information. + $objectService = $this->getObjectService(); + if ($objectService === null) { + return [ + 'success' => false, + 'error' => 'ObjectService not available', + ]; } - } - // OPTIMIZATION: Build pattern cache on first encounter of section type. - $patterns = $this->buildIdentifierPatternsForSection(sectionName: $sectionName); - $this->identifierPatternCache[$sectionName] = $patterns; + // Get object counts using the proper getter methods. + $elementObjects = $this->getElementObjects(); + $organizationObjects = $this->getOrganizationObjects(); + $viewObjects = $this->getViewObjects(); + $relationshipObjects = $this->getRelationshipObjects(); + $modelObjects = $this->getModelObjects(); + $propertyObjects = $this->getPropertyObjects(); + $propertyDefinitionObjects = $this->getPropertyDefinitionObjects(); - // Try all patterns and return first successful match. - foreach ($patterns as $pattern) { - $result = $this->extractIdentifierByPattern(item: $item, pattern: $pattern); - if ($result !== null) { - return $result; - } - } + // Calculate totals. + $elemCount = count($elementObjects) + count($organizationObjects); + $viewRelCount = count($viewObjects) + count($relationshipObjects); + $modelPropCount = count($modelObjects) + count($propertyObjects); + $totalCount = $elemCount + $viewRelCount + $modelPropCount + count($propertyDefinitionObjects); - return null; - }//end extractIdentifier() + return [ + 'success' => true, + 'status' => 'ready', + 'model_count' => count($modelObjects), + 'total_objects' => $totalCount, + 'element_count' => count($elementObjects), + 'organization_count' => count($organizationObjects), + 'view_count' => count($viewObjects), + 'relationship_count' => count($relationshipObjects), + 'property_count' => count($propertyObjects), + 'property_definition_count' => count($propertyDefinitionObjects), + ]; + } catch (\Exception $e) { + $this->logger->error( + 'Failed to get ArchiMate status', + [ + 'error' => $e->getMessage(), + ] + ); + + return [ + 'success' => false, + 'error' => $e->getMessage(), + ]; + }//end try + }//end getArchiMateStatus() /** - * OPTIMIZATION: Extract identifier using a specific pattern - * - * @param array $item The item to extract from - * @param array $pattern The extraction pattern ['path' => string[], 'type' => string] + * Get AMEF register ID from configuration * - * @return string|null The extracted identifier or null + * @return int|null The register ID or null if not configured */ - private function extractIdentifierByPattern(array $item, array $pattern): ?string { - $path = $pattern['path']; - $type = $pattern['type']; - - // Navigate to the target location. - $current = $item; - foreach ($path as $key) { - if (isset($current[$key]) === false) { - return null; + private function getAmefRegisterId(): ?int { + // Retrieve AMEF configuration (use SettingsService for consistency). + $amefConfig = $this->settingsService->getAmefConfig(); + + // Try JSON config keys first: support both 'register_id' and 'register'. + $rawRegisterId = $amefConfig['register_id'] ?? $amefConfig['register'] ?? null; + + // Fallback to legacy individual app config keys if not present in JSON. + if ($rawRegisterId === null || $rawRegisterId === '') { + $rawRegisterId = $this->config->getValueString('stackiq', 'amef_register_id', ''); + if ($this->config->getValueString('stackiq', 'amef_register', '') !== '') { + // If only the plain register key is configured, use it as the ID. + $rawRegisterId = $this->config->getValueString('stackiq', 'amef_register', ''); } + } - $current = $current[$key]; + // Validate and normalize to positive int. + if ($rawRegisterId !== '' && is_numeric((string)$rawRegisterId) === true) { + $registerId = (int)$rawRegisterId; + if ($registerId > 0) { + return $registerId; + } } - // Extract based on type. - switch ($type) { - case 'direct': - if (is_string($current) === true) { - return $current; - } - return null; - case 'value': - if (is_array($current) === true && isset($current['_value']) === true) { - return $current['_value']; - } - return null; - case 'array_search': - if (is_array($current) === true) { - foreach ($current as $childItem) { - if (isset($childItem['_attributes']['identifierRef']) === true) { - return (string)$childItem['_attributes']['identifierRef']; - } - } - } - return null; - default: - return null; - }//end switch - }//end extractIdentifierByPattern() + return null; + }//end getAmefRegisterId() /** - * OPTIMIZATION: Build identifier extraction patterns for a section type + * Get element objects from the database * - * @param string $sectionName The section name + * @param array $query Query parameters * - * @return array Array of extraction patterns ordered by likelihood of success + * @return array Array of element objects */ - private function buildIdentifierPatternsForSection(string $sectionName): array { - $patterns = []; - - // Special handling for organizations. - if ($sectionName === 'organizations') { - $patterns[] = ['path' => ['_attributes', 'identifierRef'], 'type' => 'direct']; - $patterns[] = ['path' => ['item'], 'type' => 'array_search']; - $patterns[] = ['path' => ['label'], 'type' => 'value']; - $patterns[] = ['path' => ['label'], 'type' => 'direct']; - } else { - // Standard patterns for other sections (ordered by frequency in ArchiMate). - $patterns[] = ['path' => ['_attributes', 'identifier'], 'type' => 'direct']; - $patterns[] = ['path' => ['_attributes', 'id'], 'type' => 'direct']; - $patterns[] = ['path' => ['identifier'], 'type' => 'value']; - $patterns[] = ['path' => ['identifier'], 'type' => 'direct']; - $patterns[] = ['path' => ['id'], 'type' => 'value']; - $patterns[] = ['path' => ['id'], 'type' => 'direct']; - $patterns[] = ['path' => ['_attributes', 'name'], 'type' => 'direct']; - $patterns[] = ['path' => ['name'], 'type' => 'value']; - $patterns[] = ['path' => ['name'], 'type' => 'direct']; - } - - return $patterns; - }//end buildIdentifierPatternsForSection() + public function getElementObjects(array $query = []): array { + return $this->getObjectsWithPagination(schemaType: 'element', query: $query); + }//end getElementObjects() /** - * Convert normalized data to OpenRegister objects with @self structure - * - * This method creates OpenRegister objects from the normalized ArchiMate data: - * 1. Creates a model object with proper @self structure - * 2. Creates section objects for each item (elements, relationships, etc.) - * 3. Ensures each object has the required @self structure for ObjectService::saveObjects - * 4. Links all objects to the parent model via model_identifier + * Get organization objects from the database * - * @param array $normalizedData Normalized ArchiMate data with model_identifier - * @param string $modelIdentifier The model identifier for linking objects + * @param array $query Query parameters * - * @return array Array of OpenRegister objects with proper @self structure - */ - private function convertToOpenRegisterObjects(array $normalizedData, string $modelIdentifier): array { - $this->logger->info( - 'Converting to OpenRegister objects with @self structure', - [ - 'model_identifier' => $modelIdentifier, - ] - ); - - $objects = []; - - // STEP 1: Convert model metadata to model object. - if (empty($normalizedData['model_metadata']) === false) { - $this->logger->debug('Creating model object from metadata'); - $objects[] = $this->createModelObject( - metadata: $normalizedData['model_metadata'], - modelIdentifier: $modelIdentifier - ); - } - - // STEP 2: Convert each section to individual objects. - $sections = ['elements', 'relationships', 'organizations', 'views', 'property_definitions']; - - // OPTIMIZATION: Removed excessive debug logging from tight loops. - $sectionCounts = []; - foreach ($sections as $section) { - if (empty($normalizedData[$section]) === false && is_array($normalizedData[$section]) === true) { - $sectionCounts[$section] = count($normalizedData[$section]); - foreach ($normalizedData[$section] as $identifier => $data) { - $objects[] = $this->createSectionObject( - section: $section, - identifier: $identifier, - data: $data - ); - } - } else { - $sectionCounts[$section] = 0; - } - } - - // Single consolidated log entry. - $this->logger->debug('Sections processed', $sectionCounts); - - $this->logger->info( - 'Conversion to OpenRegister objects completed', - [ - 'model_identifier' => $modelIdentifier, - 'total_objects' => count($objects), - 'sections_processed' => $sections, - ] - ); - - return $objects; - }//end convertToOpenRegisterObjects() - - /** - * Create model object with @self structure - * - * @param array $metadata Model metadata - * @param string $modelIdentifier Model identifier - * - * @return array Model object with @self structure - */ - private function createModelObject(array $metadata, string $modelIdentifier): array { - // OPTIMIZATION: Use cached configuration values. - $registerId = $this->cachedConfig['registerId']; - $schemaId = $this->cachedConfig['schemaIds']['model']; - - // Create object with @self structure and metadata at root level (no JSON serialization). - $object = [ - '@self' => [ - 'register' => $registerId, - 'schema' => $schemaId, - 'id' => $metadata['identifier'] ?? uniqid('model_'), - 'published' => date('Y-m-d\TH:i:s\Z'), - ], - 'identifier' => $metadata['identifier'] ?? '', - 'section' => 'model', - 'model_identifier' => $modelIdentifier, - ]; - - // Merge metadata directly at root level. - return array_merge($object, $metadata); - }//end createModelObject() - - /** - * Create section object with @self structure and flattened XML data - * - * @param string $section Section name - * @param string $identifier Item identifier - * @param array $data Item data (already contains XML data at root level) - * - * @return array Section object with @self structure - */ - private function createSectionObject(string $section, string $identifier, array $data): array { - // OPTIMIZATION: Use cached configuration values. - $registerId = $this->cachedConfig['registerId']; - $schemaId = $this->cachedConfig['schemaIds'][$section] ?? $this->getSchemaIdForSection(section: $section); - - // Create object with @self structure and XML data at root level (no double serialization). - $object = [ - '@self' => [ - 'register' => $registerId, - 'schema' => $schemaId, - 'id' => $identifier, - 'published' => date('Y-m-d\TH:i:s\Z'), - ], - ]; - - // Set slug: first try from _slug field, then from Object ID property, then extract from identifier. - $slug = null; - - // Check if there's a temporary slug to move to @self structure. - if (isset($data['_slug']) === true) { - $slug = $data['_slug']; - unset($data['_slug']); - } elseif (isset($data['Object ID']) === true) { - // Check if we have "Object ID" property directly. - $slug = $data['Object ID']; - } elseif (str_starts_with($identifier, 'id-') === true) { - // Fallback: extract from identifier (remove "id-" prefix if present). - $slug = substr($identifier, 3); - } - - // Set the slug if we found one. - if (empty($slug) === false) { - $object['@self']['slug'] = $slug; - } - - // Merge XML data directly at root level (data already contains identifier, section, model_identifier). - return array_merge($object, $data); - }//end createSectionObject() - - /** - * Save objects to database using ObjectService::saveObjects - * - * @param array $objects Objects to save - * - * @return array Saved objects - */ - private function saveObjectsToDatabase(array $objects): array { - $saveStartTime = microtime(true); - - $serviceInitStartTime = microtime(true); - $objectService = $this->getObjectService(); - if ($objectService === null) { - throw new \RuntimeException('ObjectService not available'); - } - - $serviceInitTime = microtime(true) - $serviceInitStartTime; - - // ENHANCEMENT: Process GEMMA Referentiecomponent-Standaard relationships before saving. - $gemmaProcessingStartTime = microtime(true); - $objects = $this->processGemmaReferenceComponentStandards(objects: $objects); - $gemmaProcessingTime = microtime(true) - $gemmaProcessingStartTime; - - $this->logger->info( - 'Saving objects to database using parallel batch processing', - [ - 'count' => count($objects), - 'batch_size' => self::PERFORMANCE_OPTIMIZATIONS['batch_size'], - 'parallel_batches' => self::PERFORMANCE_OPTIMIZATIONS['parallel_batches'], - 'service_init_time' => round($serviceInitTime, 3), - 'gemma_processing_time' => round($gemmaProcessingTime, 3), - ] - ); - - // OPTIMIZATION: Use cached register ID. - $registerId = $this->cachedConfig['registerId']; - - // PERFORMANCE OPTIMIZATION: Use parallel batch processing for large datasets. - $batchProcessingStartTime = microtime(true); - // PERFORMANCE_OPTIMIZATIONS['parallel_processing'] is a class constant set - // to true, so only the batch-size threshold decides this. - if (count($objects) > self::PERFORMANCE_OPTIMIZATIONS['batch_size']) { - $result = $this->saveObjectsInParallelBatches( - objects: $objects, - objectService: $objectService, - registerId: $registerId - ); - } else { - // Fallback to single batch for small datasets. - $result = $this->saveObjectsInSingleBatch( - objects: $objects, - objectService: $objectService, - registerId: $registerId - ); - } - - $batchProcessingTime = microtime(true) - $batchProcessingStartTime; - - $totalSaveTime = microtime(true) - $saveStartTime; - - $this->logger->info( - 'Database save operation completed', - [ - 'total_save_time' => round($totalSaveTime, 3), - 'service_init_time' => round($serviceInitTime, 3), - 'gemma_processing_time' => round($gemmaProcessingTime, 3), - 'batch_processing_time' => round($batchProcessingTime, 3), - 'objects_saved' => count($result), - 'save_rate_objects_per_second' => round(count($objects) / max($totalSaveTime, 0.001), 1), - ] - ); - - // Store timing breakdown for performance metrics. - $this->lastSaveTimingBreakdown = [ - 'total_save_seconds' => round($totalSaveTime, 3), - 'service_init_seconds' => round($serviceInitTime, 3), - 'gemma_processing_seconds' => round($gemmaProcessingTime, 3), - 'batch_processing_seconds' => round($batchProcessingTime, 3), - 'objects_saved' => count($result), - 'save_rate_objects_per_second' => round(count($objects) / max($totalSaveTime, 0.001), 1), - ]; - - return $result; - }//end saveObjectsToDatabase() - - /** - * Save objects in parallel batches for maximum performance - * - * @param array $objects Array of objects to save - * @param ObjectServiceInterface $objectService ObjectService instance - * @param int $registerId Register ID - * - * @return array Array of saved objects - */ - private function saveObjectsInParallelBatches(array $objects, ObjectServiceInterface $objectService, int $registerId): array { - $batchSize = self::PERFORMANCE_OPTIMIZATIONS['batch_size']; - $parallelBatches = self::PERFORMANCE_OPTIMIZATIONS['parallel_batches']; - - // INTELLIGENT BATCH SIZING: Create size-aware batches instead of fixed-size chunks. - $chunks = $this->createIntelligentBatches(objects: $objects); - $totalChunks = count($chunks); - - $this->logger->info( - 'Starting intelligent batch processing', - [ - 'total_objects_to_save' => count($objects), - 'intelligent_batches_created' => $totalChunks, - 'batch_sizes' => array_map('count', $chunks), - 'batching_method' => 'size_aware_intelligent', - 'mysql_packet_limit_safe' => true, - ] - ); - - $allResults = []; - $processedChunks = 0; - - // Accumulate statistics from all chunks. - $aggregatedStats = [ - 'saved' => [], - 'updated' => [], - 'skipped' => [], - 'invalid' => [], - ]; - - // Process chunks sequentially but with larger batch sizes for better performance. - foreach ($chunks as $chunkIndex => $chunk) { - // OPTIMIZATION: Removed debug logging from chunk processing loop. - try { - if (self::PERFORMANCE_OPTIMIZATIONS['disable_rbac'] === true) { - $rbacValue = false; - } else { - $rbacValue = true; - } - - $saveResult = $objectService->saveObjects( - objects: $chunk, - register: $registerId, - schema: null, - _rbac: $rbacValue, - validation: !self::PERFORMANCE_OPTIMIZATIONS['disable_validation'], - events: !self::PERFORMANCE_OPTIMIZATIONS['disable_events'] - ); - - // Accumulate statistics from this chunk. - $aggregatedStats['saved'] = array_merge($aggregatedStats['saved'], $saveResult['saved'] ?? []); - $aggregatedStats['updated'] = array_merge($aggregatedStats['updated'], $saveResult['updated'] ?? []); - $aggregatedStats['skipped'] = array_merge($aggregatedStats['skipped'], $saveResult['skipped'] ?? []); - $aggregatedStats['invalid'] = array_merge($aggregatedStats['invalid'], $saveResult['invalid'] ?? []); - - $savedObjects = array_merge( - $saveResult['saved'] ?? [], - $saveResult['updated'] ?? [] - ); - - $allResults = array_merge($allResults, $savedObjects); - - $processedChunks++; - $this->logger->info( - 'Processed chunk', - [ - 'processed_chunks' => $processedChunks, - 'total_chunks' => $totalChunks, - 'progress_percent' => round(($processedChunks / $totalChunks) * 100, 1), - 'chunk_saved' => count($saveResult['saved'] ?? []), - 'chunk_updated' => count($saveResult['updated'] ?? []), - 'chunk_skipped' => count($saveResult['skipped'] ?? []), - 'chunk_invalid' => count($saveResult['invalid'] ?? []), - ] - ); - } catch (\Exception $e) { - $this->logger->error( - 'Error processing chunk', - [ - 'chunk_index' => $chunkIndex, - 'error' => $e->getMessage(), - ] - ); - // Continue with other chunks. - }//end try - - // Memory cleanup between chunks. - // PERFORMANCE_OPTIMIZATIONS['memory_cleanup'] is a class constant set - // to true, so this was never conditional. - $this->cleanupMemory(); - }//end foreach - - // Store the aggregated result for statistics calculation. - $this->lastSaveResult = $aggregatedStats; - - $this->logger->info( - 'Optimized batch processing completed', - [ - 'total_objects_processed' => count($allResults), - 'total_chunks_processed' => $totalChunks, - 'aggregated_saved' => count($aggregatedStats['saved']), - 'aggregated_updated' => count($aggregatedStats['updated']), - 'aggregated_skipped' => count($aggregatedStats['skipped']), - 'aggregated_invalid' => count($aggregatedStats['invalid']), - ] - ); - - return $allResults; - }//end saveObjectsInParallelBatches() - - /** - * Save objects in a single batch (fallback method) - * - * @param array $objects Array of objects to save - * @param ObjectServiceInterface $objectService ObjectService instance - * @param int $registerId Register ID - * - * @return array Array of saved objects - */ - private function saveObjectsInSingleBatch(array $objects, ObjectServiceInterface $objectService, int $registerId): array { - $this->logger->info( - 'Using single batch processing', - [ - 'count' => count($objects), - ] - ); - - if (self::PERFORMANCE_OPTIMIZATIONS['disable_rbac'] === true) { - $rbacValue = false; - } else { - $rbacValue = true; - } - - $saveResult = $objectService->saveObjects( - objects: $objects, - register: $registerId, - schema: null, - _rbac: $rbacValue, - validation: !self::PERFORMANCE_OPTIMIZATIONS['disable_validation'], - events: !self::PERFORMANCE_OPTIMIZATIONS['disable_events'] - ); - - // Store the save result for later access to statistics. - $this->lastSaveResult = $saveResult; - - // Extract saved objects from the new structured return format. - $savedObjects = array_merge( - $saveResult['saved'] ?? [], - $saveResult['updated'] ?? [] - ); - - // Log detailed results including validation errors. - $this->logger->info( - 'Objects saved successfully', - [ - 'saved_count' => count($saveResult['saved'] ?? []), - 'updated_count' => count($saveResult['updated'] ?? []), - 'unchanged_count' => count($saveResult['skipped'] ?? []), - 'invalid_count' => count($saveResult['invalid'] ?? []), - 'error_count' => count($saveResult['errors'] ?? []), - 'total_processed' => $saveResult['statistics']['totalProcessed'] ?? 0, - ] - ); - - // Log any validation errors for debugging. - if (empty($saveResult['invalid']) === false) { - foreach ($saveResult['invalid'] as $invalidItem) { - $this->logger->warning( - 'Object failed validation during import', - [ - 'object_id' => $invalidItem['object']['@self']['id'] ?? 'unknown', - 'error' => $invalidItem['error'] ?? 'Unknown validation error', - 'type' => $invalidItem['type'] ?? 'ValidationException', - ] - ); - } - } - - // Log details about skipped objects if any. - if (empty($saveResult['skipped']) === false) { - $this->logger->info( - 'Objects skipped during import (no changes detected)', - [ - 'skipped_count' => count($saveResult['skipped']), - 'sample_skipped_ids' => array_slice( - array_map(fn ($obj) => $obj->getUuid() ?? 'unknown', $saveResult['skipped']), - 0, - 5 - ), - ] - ); - } - - // Return the combined saved and updated objects (maintaining backward compatibility). - return $savedObjects; - }//end saveObjectsInSingleBatch() - - /** - * Get ObjectService from container - * - * @return ObjectServiceInterface|null ObjectService instance or null if not available - */ - private function getObjectService(): ?ObjectServiceInterface { - if ($this->appManager->isInstalled(appId: 'openregister') === false) { - return null; - } - - try { - return $this->container->get(ObjectService::class); - } catch (\Exception $e) { - $this->logger->warning( - 'Failed to get ObjectService', - [ - 'error' => $e->getMessage(), - ] - ); - return null; - } - }//end getObjectService() - - /** - * Initialize cached configuration values for performance optimization - * - * @return void - */ - private function initializeCache(): void { - if ($this->cachedConfig !== null) { - return; - // Already cached. - } - - // Get the AMEF register ID from configuration - throw error if missing. - $amefConfig = $this->settingsService->getAmefConfig(); - if (isset($amefConfig['register']) === false || empty($amefConfig['register']) === true) { - throw new \InvalidArgumentException( - 'AMEF register ID is not configured. Please configure the AMEF register via the admin interface.' - ); - } - - $registerId = (int)$amefConfig['register']; - - $this->cachedConfig = [ - 'registerId' => $registerId, - // Use AMEF register ID directly. - 'schemaIds' => [ - 'model' => $this->settingsService->getSchemaIdForObjectType('model'), - 'element' => $this->settingsService->getSchemaIdForObjectType('element'), - 'relationship' => $this->settingsService->getSchemaIdForObjectType('relationship'), - 'view' => $this->settingsService->getSchemaIdForObjectType('view'), - 'organization' => $this->settingsService->getSchemaIdForObjectType('organization'), - 'property_definition' => $this->settingsService->getSchemaIdForObjectType('property_definition'), - // NOTE: 'property' removed - properties are never root-level - // AMEF objects, only nested within other elements. - ], - ]; - - $this->logger->debug( - 'ArchiMateService: Cache initialized', - [ - 'registerId' => $this->cachedConfig['registerId'], - 'schemaIds' => $this->cachedConfig['schemaIds'], - ] - ); - - // Validate that all required schema IDs are configured. - $this->validateRequiredConfiguration(); - }//end initializeCache() - - /** - * Validates that all required configuration is present before import. - * - * @return void - * - * @throws \RuntimeException If required configuration is missing. - */ - private function validateRequiredConfiguration(): void { - $missingConfig = []; - $requiredSchemaTypes = ['model', 'element', 'relationship', 'view', 'organization', 'property']; - - // Check register ID. - if (empty($this->cachedConfig['registerId']) === true) { - $missingConfig[] = 'AMEF Register ID (amef_register)'; - } - - // Check all required schema IDs. - foreach ($requiredSchemaTypes as $schemaType) { - $schemaId = $this->cachedConfig['schemaIds'][$schemaType] ?? null; - if ($schemaId === null) { - $configKey = "amef_{$schemaType}_schema"; - $missingConfig[] = "Schema ID for {$schemaType} ({$configKey})"; - } - } - - // If any configuration is missing, throw detailed error. - if (empty($missingConfig) === false) { - $this->logger->error( - 'ArchiMateService: Missing required configuration', - [ - 'missing_config' => $missingConfig, - 'current_config' => [ - 'registerId' => $this->cachedConfig['registerId'], - 'schemaIds' => $this->cachedConfig['schemaIds'], - ], - ] - ); - - $errorMessage = 'ArchiMate import cannot proceed due to missing configuration:' . "\n\n"; - $errorMessage .= "Missing configuration:\n"; - foreach ($missingConfig as $item) { - $errorMessage .= "- {$item}\n"; - } - - $settingsHint = 'in the Stackiq settings before importing.'; - $errorMessage .= "\nPlease configure the AMEF register and all required schema IDs $settingsHint"; - $manualHint = 'or set them manually via the admin interface.'; - $errorMessage .= "\nYou can use the auto-configuration feature $manualHint"; - - throw new \RuntimeException($errorMessage); - }//end if - - $this->logger->info( - 'ArchiMateService: Configuration validation passed', - [ - 'registerId' => $this->cachedConfig['registerId'], - 'configuredSchemas' => count(array_filter($this->cachedConfig['schemaIds'])), - ] - ); - }//end validateRequiredConfiguration() - - /** - * Log current memory usage for performance monitoring - * - * @param string $stage Description of the current processing stage - * - * @return void - */ - private function logMemoryUsage(string $stage): void { - // Check if debug logging is available (Nextcloud logger doesn't have isDebug method). - $memoryUsage = memory_get_usage(true); - $memoryPeak = memory_get_peak_usage(true); - $memoryLimit = ini_get('memory_limit'); - - $this->logger->debug( - "Memory usage at: {$stage}", - [ - 'current_mb' => round($memoryUsage / 1024 / 1024, 2), - 'peak_mb' => round($memoryPeak / 1024 / 1024, 2), - 'limit' => $memoryLimit, - ] - ); - }//end logMemoryUsage() - - /** - * Clean up memory by forcing garbage collection - * - * @return void - */ - private function cleanupMemory(): void { - if (function_exists('gc_collect_cycles') === true) { - $cycles = gc_collect_cycles(); - // Use PSR-3 standard logging instead of isDebug() check. - $this->logger->debug( - 'Garbage collection completed', - [ - 'cycles_collected' => $cycles, - ] - ); - } - }//end cleanupMemory() - - /** - * NOTE: Removed deprecated methods getArchiMateRegisterId() and getArchiMateModelSchemaId() - * that had hardcoded fallbacks. All configuration is now retrieved via SettingsService - * through the initializeCache() method and proper AMEF configuration. - */ - - /** - * Get schema ID for a section - * - * @param string $section Section name - * - * @return int Schema ID - */ - private function getSchemaIdForSection(string $section): int { - // Map section names to object types for SettingsService. - $objectTypeMapping = [ - 'elements' => 'element', - 'relationships' => 'relationship', - 'views' => 'view', - 'organizations' => 'organization', - 'property_definitions' => 'property_definition', - ]; - - $objectType = $objectTypeMapping[$section] ?? $section; - $schemaId = $this->settingsService->getSchemaIdForObjectType($objectType); - - // Ensure schema ID is configured - no hardcoded fallbacks. - if ($schemaId === null) { - $configMsg = "Schema ID for section '{$section}' is not configured."; - $helpMsg = 'Please configure all AMEF schema IDs via the admin interface.'; - $typeMsg = "Expected object type: '{$objectType}'"; - throw new \RuntimeException("{$configMsg} {$helpMsg} {$typeMsg}"); - } - - return $schemaId; - }//end getSchemaIdForSection() - - /** - * Read a configured id, failing closed on the empty default. - * - * The legacy fallback used to read every id with `''` as its default, so - * an unconfigured instance produced a config array full of empty STRINGS. - * Its consumers guard with `=== null` — `ViewService::getViews()` and - * `getView()` throw only when a value `=== null` — and `'' === null` is - * false. Today nothing reaches a query only because that fallback writes - * PLURAL key names (`views_schema`) while every consumer reads SINGULAR - * ones (`view_schema`), so the lookups miss and fall back to `null`. That - * is an accident of naming, not a defence: adding the singular keys — the - * obvious "cleanup" — would send `register => ''` straight into - * `searchObjects()` as an UNPINNED query, and an unpinned query returns - * rows, which reads exactly like a correct result. - * - * Returning null instead of `''` makes `?? null` downstream yield null, - * which is what every consumer already checks for, and the warning names - * the missing key so a misconfigured import stops reporting "0 objects" - * with no explanation. - * - * @param string $key The app-config key holding the id. - * - * @return string|null The configured id, or null when it is unset. - * - * @spec openspec/specs/archimate-import/spec.md - */ - private function resolveConfiguredId(string $key): ?string { - $value = $this->config->getValueString('stackiq', $key, ''); - if (trim($value) === '') { - $this->logger->warning( - 'ArchiMate configuration is incomplete — this id is not configured, so it is omitted rather than passed on as an empty string', - ['key' => $key] - ); - - return null; - } - - return $value; - }//end resolveConfiguredId() - - /** - * Get AMEF configuration from app config - * - * @return array AMEF configuration - * @spec openspec/specs/archimate-import/spec.md - */ - public function getAmefConfig(): array { - $this->logger->info('Getting AMEF configuration'); - - try { - // Get configuration from app config using the correct method. - $config = $this->config->getValueString('stackiq', 'amef_config', '{}'); - $decoded = json_decode($config, true); - - if (is_array($decoded) === false) { - // Fallback to individual config values for backward - // compatibility. Every id is read through - // resolveConfiguredId(), which guards the empty default at the - // point of the read and omits the key entirely when it is - // unset — so `?? null` downstream yields null, which is what - // the consumers already check for. - $decoded = array_filter( - [ - 'register_id' => $this->resolveConfiguredId(key: 'amef_register'), - 'model_schema_id' => $this->resolveConfiguredId(key: 'amef_model_schema'), - 'elements_schema' => $this->resolveConfiguredId(key: 'amef_elements_schema'), - 'relationships_schema' => $this->resolveConfiguredId(key: 'amef_relationships_schema'), - 'views_schema' => $this->resolveConfiguredId(key: 'amef_views_schema'), - 'organizations_schema' => $this->resolveConfiguredId(key: 'amef_organizations_schema'), - 'folders_schema' => $this->resolveConfiguredId(key: 'amef_folders_schema'), - 'property_definitions_schema' => $this->resolveConfiguredId(key: 'amef_property_definitions_schema'), - ], - static function ($value) { - return $value !== null; - } - ); - }//end if - - return $decoded; - } catch (\Exception $e) { - $this->logger->error( - 'Failed to get AMEF configuration', - [ - 'error' => $e->getMessage(), - ] - ); - - return [ - 'success' => false, - 'error' => $e->getMessage(), - ]; - }//end try - }//end getAmefConfig() - - /** - * Get the current status of ArchiMate operations - * - * @return array Status information including import/export status and object counts - * @spec openspec/specs/archimate-import/spec.md - */ - public function getArchiMateStatus(): array { - $this->logger->info('Getting ArchiMate status'); - - try { - // Get basic status information. - $objectService = $this->getObjectService(); - if ($objectService === null) { - return [ - 'success' => false, - 'error' => 'ObjectService not available', - ]; - } - - // Get object counts using the proper getter methods. - $elementObjects = $this->getElementObjects(); - $organizationObjects = $this->getOrganizationObjects(); - $viewObjects = $this->getViewObjects(); - $relationshipObjects = $this->getRelationshipObjects(); - $modelObjects = $this->getModelObjects(); - $propertyObjects = $this->getPropertyObjects(); - $propertyDefinitionObjects = $this->getPropertyDefinitionObjects(); - - // Calculate totals. - $elemCount = count($elementObjects) + count($organizationObjects); - $viewRelCount = count($viewObjects) + count($relationshipObjects); - $modelPropCount = count($modelObjects) + count($propertyObjects); - $totalCount = $elemCount + $viewRelCount + $modelPropCount + count($propertyDefinitionObjects); - - return [ - 'success' => true, - 'status' => 'ready', - 'model_count' => count($modelObjects), - 'total_objects' => $totalCount, - 'element_count' => count($elementObjects), - 'organization_count' => count($organizationObjects), - 'view_count' => count($viewObjects), - 'relationship_count' => count($relationshipObjects), - 'property_count' => count($propertyObjects), - 'property_definition_count' => count($propertyDefinitionObjects), - ]; - } catch (\Exception $e) { - $this->logger->error( - 'Failed to get ArchiMate status', - [ - 'error' => $e->getMessage(), - ] - ); - - return [ - 'success' => false, - 'error' => $e->getMessage(), - ]; - }//end try - }//end getArchiMateStatus() - - /** - * Get AMEF register ID from configuration - * - * @return int|null The register ID or null if not configured - */ - private function getAmefRegisterId(): ?int { - // Retrieve AMEF configuration (use SettingsService for consistency). - $amefConfig = $this->settingsService->getAmefConfig(); - - // Try JSON config keys first: support both 'register_id' and 'register'. - $rawRegisterId = $amefConfig['register_id'] ?? $amefConfig['register'] ?? null; - - // Fallback to legacy individual app config keys if not present in JSON. - if ($rawRegisterId === null || $rawRegisterId === '') { - $rawRegisterId = $this->config->getValueString('stackiq', 'amef_register_id', ''); - if ($this->config->getValueString('stackiq', 'amef_register', '') !== '') { - // If only the plain register key is configured, use it as the ID. - $rawRegisterId = $this->config->getValueString('stackiq', 'amef_register', ''); - } - } - - // Validate and normalize to positive int. - if ($rawRegisterId !== '' && is_numeric((string)$rawRegisterId) === true) { - $registerId = (int)$rawRegisterId; - if ($registerId > 0) { - return $registerId; - } - } - - return null; - }//end getAmefRegisterId() - - /** - * Get AMEF schema ID for a specific ArchiMate type via SettingsService - * - * This method retrieves the schema ID for a given ArchiMate type from SettingsService. - * - * @param string $archiMateType The ArchiMate type (e.g., 'element', 'organization', 'relationship') - * - * @return int|null The schema ID for the given type or null if not configured - */ - private function getAmefSchemaIdForType(string $archiMateType): ?int { - // Use SettingsService to get schema ID. - $schemaId = $this->settingsService->getSchemaIdForObjectType($archiMateType); - return $schemaId; - }//end getAmefSchemaIdForType() - - /** - * Get element objects from the database - * - * @param array $query Query parameters - * - * @return array Array of element objects - */ - public function getElementObjects(array $query = []): array { - return $this->getObjectsWithPagination(schemaType: 'element', query: $query); - }//end getElementObjects() - - /** - * Get organization objects from the database - * - * @param array $query Query parameters - * - * @return array Array of organization objects + * @return array Array of organization objects */ public function getOrganizationObjects(array $query = []): array { return $this->getObjectsWithPagination(schemaType: 'organization', query: $query); @@ -1859,784 +875,113 @@ private function getObjectsWithPagination(string $schemaType, array $query = []) return []; } - // Extract pagination parameters. - $limit = $query['limit'] ?? 1000; - // Default limit for large datasets. - $offset = $query['offset'] ?? 0; - $usePagination = $query['use_pagination'] ?? false; - - // Remove pagination parameters from query. - unset($query['limit'], $query['offset'], $query['use_pagination']); - - // Build base query for register and schema. - $baseQuery = [ - '@self' => [ - 'register' => (int)$registerId, - 'schema' => (int)$schemaId, - ], - ]; - - // Merge with provided query. - $finalQuery = array_merge_recursive($baseQuery, $query); - - // Add pagination if requested. - if ($usePagination !== false && $limit > 0) { - $finalQuery['@pagination'] = [ - 'limit' => (int)$limit, - 'offset' => (int)$offset, - ]; - } - - // Always bound the actual query: OpenRegister's searchObjects() - // only reads the top-level `_limit`/`_offset` keys (see - // MagicMapper/MagicSearchHandler) — `@pagination` above is not - // read by the query layer, so this call was unbounded regardless - // of the `usePagination` toggle. `$limit` already defaults to - // 1000 above, so this is a straight bound, not a behavior change. - $finalQuery['_limit'] = (int)$limit; - if ($usePagination !== false) { - $finalQuery['_offset'] = (int)$offset; - } - - $paginationValue = 'disabled'; - if ($usePagination === true) { - $paginationValue = 'enabled'; - } - - $this->logger->debug( - "ArchiMateService: Retrieving {$schemaType} objects", - [ - 'register' => $registerId, - 'schema' => $schemaId, - 'query' => $finalQuery, - 'pagination' => $paginationValue, - ] - ); - - // Use searchObjects method for filtering. - $objects = $objectService->searchObjects($finalQuery); - - $paginationValue = 'disabled'; - if ($usePagination === true) { - $paginationValue = 'enabled'; - } - - $this->logger->debug( - "ArchiMateService: Retrieved {$schemaType} objects", - [ - 'register' => $registerId, - 'schema' => $schemaId, - 'count' => count($objects), - 'pagination' => $paginationValue, - ] - ); - - return $objects; - } catch (\Exception $e) { - $this->logger->error( - "ArchiMateService: Failed to retrieve {$schemaType} objects", - [ - 'error' => $e->getMessage(), - 'trace' => $e->getTraceAsString(), - ] - ); - - return []; - }//end try - }//end getObjectsWithPagination() - - /** - * Check if import is in progress - * - * @return bool True if import is in progress - * @spec openspec/specs/archimate-import/spec.md - */ - public function isImportInProgress(): bool { - // For now, return false as we haven't implemented status tracking yet. - return false; - }//end isImportInProgress() - - /** - * Check if export is in progress - * - * @return bool True if export is in progress - * @spec openspec/specs/archimate-import/spec.md - */ - public function isExportInProgress(): bool { - // For now, return false as we haven't implemented status tracking yet. - return false; - }//end isExportInProgress() - - /** - * Check if any operation is in progress - * - * @return bool True if any operation is in progress - */ - private function isOperationInProgress(): bool { - return $this->isImportInProgress() || $this->isExportInProgress(); - }//end isOperationInProgress() - - /** - * Create intelligent batches based on object size to prevent MySQL packet size issues - * - * This method analyzes object sizes and creates batches that stay under the MySQL - * max_allowed_packet limit while maintaining reasonable performance. - * - * @param array $objects Array of objects to batch - * - * @return array Array of batches, each containing objects that fit within size limits - */ - private function createIntelligentBatches(array $objects): array { - $maxBatchSizeBytes = self::PERFORMANCE_OPTIMIZATIONS['max_batch_size_bytes']; - $minBatchSize = self::PERFORMANCE_OPTIMIZATIONS['min_batch_size']; - $sampleSize = self::PERFORMANCE_OPTIMIZATIONS['size_estimation_sample']; - - if (empty($objects) === true) { - return []; - } - - // Estimate average object size by sampling. - $avgObjectSize = $this->estimateAverageObjectSize(objects: $objects, sampleSize: $sampleSize); - - // Calculate optimal batch size based on object size. - $optimalBatchSize = max($minBatchSize, intval($maxBatchSizeBytes / $avgObjectSize)); - - $this->logger->info( - 'Intelligent batch sizing analysis', - [ - 'total_objects' => count($objects), - 'estimated_avg_object_size_bytes' => $avgObjectSize, - 'max_batch_size_bytes' => $maxBatchSizeBytes, - 'calculated_optimal_batch_size' => $optimalBatchSize, - 'min_batch_size_enforced' => $minBatchSize, - ] - ); - - // Create batches with size awareness. - $batches = []; - $currentBatch = []; - $currentBatchSize = 0; - - foreach ($objects as $object) { - $objectSize = $this->estimateObjectSize(object: $object); - - // Check if adding this object would exceed the batch size limit. - if (empty($currentBatch) === false && ($currentBatchSize + $objectSize) > $maxBatchSizeBytes) { - // Current batch is full, save it and start a new one. - $batches[] = $currentBatch; - $currentBatch = [$object]; - $currentBatchSize = $objectSize; - } else { - // Add object to current batch. - $currentBatch[] = $object; - $currentBatchSize += $objectSize; - } - - // Safety check: if a single object is larger than max batch size,. - // create a batch with just that object. - if (count($currentBatch) === 1 && $objectSize > $maxBatchSizeBytes) { - $this->logger->warning( - 'Very large object detected, creating single-object batch', - [ - 'object_id' => $object['@self']['id'] ?? 'unknown', - 'object_size_bytes' => $objectSize, - 'max_batch_size_bytes' => $maxBatchSizeBytes, - ] - ); - $batches[] = $currentBatch; - $currentBatch = []; - $currentBatchSize = 0; - } - }//end foreach - - // Add the last batch if it has objects. - if (empty($currentBatch) === false) { - $batches[] = $currentBatch; - } - - $this->logger->info( - 'Intelligent batching completed', - [ - 'total_objects' => count($objects), - 'total_batches_created' => count($batches), - 'batch_sizes' => array_map('count', $batches), - 'estimated_batch_sizes_bytes' => array_map( - fn ($batch) => array_sum( - array_map([$this, 'estimateObjectSize'], $batch) - ), - $batches - ), - ] - ); - - return $batches; - }//end createIntelligentBatches() - - /** - * Estimate the average size of objects by sampling - * - * @param array $objects Array of objects to sample - * @param int $sampleSize Number of objects to sample for size estimation - * - * @return int Estimated average object size in bytes - */ - private function estimateAverageObjectSize(array $objects, int $sampleSize): int { - $totalObjects = count($objects); - if ($totalObjects === 0) { - return 1000; - // Default fallback size. - } - - // Sample evenly distributed objects. - $sampleIndices = []; - if ($totalObjects <= $sampleSize) { - // Use all objects if we have fewer than sample size. - $sampleIndices = range(0, $totalObjects - 1); - } else { - // Sample evenly across the array. - $step = max(1, intval($totalObjects / $sampleSize)); - for ($i = 0; $i < $totalObjects; $i += $step) { - $sampleIndices[] = $i; - if (count($sampleIndices) >= $sampleSize) { - break; - } - } - } - - // Calculate sizes of sampled objects. - $totalSampleSize = 0; - foreach ($sampleIndices as $index) { - $totalSampleSize += $this->estimateObjectSize(object: $objects[$index]); - } - - $averageSize = intval($totalSampleSize / count($sampleIndices)); - - $this->logger->debug( - 'Object size estimation completed', - [ - 'total_objects' => $totalObjects, - 'sampled_objects' => count($sampleIndices), - 'total_sample_size_bytes' => $totalSampleSize, - 'estimated_average_size_bytes' => $averageSize, - ] - ); - - return max(1000, $averageSize); - // Minimum 1KB per object. - }//end estimateAverageObjectSize() - - /** - * Estimate the serialized size of an object for batching purposes - * - * @param array $object The object to estimate size for - * - * @return int Estimated size in bytes - */ - private function estimateObjectSize(array $object): int { - // Quick estimation based on JSON serialization. - // This includes overhead for SQL parameters and structure. - $jsonSize = strlen(json_encode($object)); - - // Add overhead for SQL INSERT statement structure. - // Each object becomes multiple parameters in a bulk INSERT. - $sqlOverhead = 500; - // Estimated overhead per object in SQL. - return $jsonSize + $sqlOverhead; - }//end estimateObjectSize() - - /** - * Calculate detailed object statistics for import operations - * - * @param array $normalizedData Normalized ArchiMate data - * - * @return array Comprehensive statistics - */ - private function calculateObjectStatistics(array $normalizedData): array { - // Initialize statistics structure. - $statistics = [ - 'elements' => ['created' => 0, 'updated' => 0, 'skipped' => 0, 'errors' => []], - 'organizations' => ['created' => 0, 'updated' => 0, 'skipped' => 0, 'errors' => []], - 'relationships' => ['created' => 0, 'updated' => 0, 'skipped' => 0, 'errors' => []], - 'views' => ['created' => 0, 'updated' => 0, 'skipped' => 0, 'errors' => []], - 'property_definitions' => ['created' => 0, 'updated' => 0, 'skipped' => 0, 'errors' => []], - ]; - - // If we have access to the actual save results from ObjectService, use those. - if ($this->lastSaveResult !== null) { - $saveResult = $this->lastSaveResult; - - // Count objects by section type from the actual processed objects. - $allProcessedObjects = array_merge( - $saveResult['saved'] ?? [], - $saveResult['updated'] ?? [], - $saveResult['skipped'] ?? [], - // For invalid objects, extract the original object from the error structure. - array_map(fn ($item) => $item['object'] ?? [], $saveResult['invalid'] ?? []) - ); - - foreach ($allProcessedObjects as $object) { - // Convert ObjectEntity to array if needed. - if (is_object($object) === true && method_exists($object, 'jsonSerialize') === true) { - $object = $object->jsonSerialize(); - } - - $sectionType = $object['section'] ?? 'elements'; - // Default to elements if section not found. - // Map section types to statistics keys. - $sectionKey = match ($sectionType) { - 'elements' => 'elements', - 'relationships' => 'relationships', - 'organizations' => 'organizations', - 'views' => 'views', - 'property_definitions' => 'property_definitions', - default => 'elements' - // Default fallback. - }; - - // No "skip unknown section types" guard: the branch above pins - // $sectionKey to a key $statistics always has, so it never fired. - - // Determine if this object was created, updated, or had errors. - $objectId = $object['@self']['id'] ?? $object['identifier'] ?? null; - - // Check if this object is in the saved (created) list. - $wasCreated = empty( - array_filter( - $saveResult['saved'] ?? [], - fn ($saved) => ($saved->getUuid() === $objectId) - ) - ) === false; - - // Check if this object is in the updated list. - $wasUpdated = empty( - array_filter( - $saveResult['updated'] ?? [], - fn ($updated) => ($updated->getUuid() === $objectId) - ) - ) === false; - - // Check if this object was skipped (no changes). - $wasSkipped = empty( - array_filter( - $saveResult['skipped'] ?? [], - fn ($skipped) => ($skipped->getUuid() === $objectId) - ) - ) === false; - - // Check if this object had validation errors. - $hasErrors = empty( - array_filter( - $saveResult['invalid'] ?? [], - fn ($invalid) => (($invalid['object']['@self']['id'] ?? null) === $objectId) - ) - ) === false; - - if ($wasCreated === true) { - $statistics[$sectionKey]['created']++; - } elseif ($wasUpdated === true) { - $statistics[$sectionKey]['updated']++; - } elseif ($wasSkipped === true) { - $statistics[$sectionKey]['skipped']++; - } elseif ($hasErrors === true) { - // Add to errors array for this section. - $errorInfo = array_filter( - $saveResult['invalid'] ?? [], - fn ($invalid) => (($invalid['object']['@self']['id'] ?? null) === $objectId) - ); - - if (empty($errorInfo) === false) { - $firstError = array_values($errorInfo)[0]; - $errorMessage = 'Unknown validation error'; - if (is_array($firstError) === true && isset($firstError['error']) === true) { - $errorMessage = $firstError['error']; - } - - $statistics[$sectionKey]['errors'][] = $errorMessage; - } - } else { - // This shouldn't happen, but leave as fallback. - $statistics[$sectionKey]['skipped']++; - }//end if - }//end foreach - } else { - // Fallback to old method if no save result is available. - $sections = ['elements', 'relationships', 'organizations', 'views', 'property_definitions']; - foreach ($sections as $section) { - if (isset($normalizedData[$section]) === true) { - $count = count($normalizedData[$section]); - // Assume all objects were created (legacy behavior). - $statistics[$section]['created'] = $count; - } - } - }//end if - - // Calculate summary totals from actual statistics. - $summary = [ - 'total_objects_created' => 0, - 'total_objects_updated' => 0, - 'total_objects_deleted' => 0, - 'total_objects_skipped' => 0, - 'total_errors' => 0, - ]; - - // No "skip the summary section" guard: `omschrijving` is written into - // $statistics on the line AFTER this loop, so the loop can never see it. - foreach ($statistics as $sectionStats) { - $summary['total_objects_created'] += $sectionStats['created']; - $summary['total_objects_updated'] += $sectionStats['updated']; - $summary['total_objects_skipped'] += $sectionStats['skipped']; - $summary['total_errors'] += count($sectionStats['errors']); - } - - $statistics['omschrijving'] = $summary; - - return $statistics; - }//end calculateObjectStatistics() - - /** - * Extract propertyDefinitions from the parsed XML and build a map - * - * @param array $data Parsed XML data - * - * @return array Map of propertyDefinitionRef => property name - */ - private function extractPropertyDefinitionMap(array $data): array { - // OPTIMIZATION: Return cached property definition map if available. - if ($this->propDefMapCache !== null) { - return $this->propDefMapCache; - } - - $map = []; - // Find propertyDefinitions section (handle possible alternative names). - $propertyDefs = null; - if (isset($data['propertyDefinitions']) === true) { - $propertyDefs = $data['propertyDefinitions']; - } elseif (isset($data['property_definitions']) === true) { - $propertyDefs = $data['property_definitions']; - } - - if ($propertyDefs !== false && isset($propertyDefs['propertyDefinition']) === true) { - $defs = $propertyDefs['propertyDefinition']; - if (isset($defs[0]) === true) { - // Array of propertyDefinition. - foreach ($defs as $def) { - if (isset($def['_attributes']['identifier']) === true && isset($def['name']) === true) { - if (is_array($def['name']) === true && isset($def['name']['_value']) === true) { - $map[$def['_attributes']['identifier']] = $def['name']['_value']; - } else { - $map[$def['_attributes']['identifier']] = $def['name']; - } - } - } - } elseif (isset($defs['_attributes']['identifier']) === true && isset($defs['name']) === true) { - // Single propertyDefinition. - if (is_array($defs['name']) === true && isset($defs['name']['_value']) === true) { - $map[$defs['_attributes']['identifier']] = $defs['name']['_value']; - } else { - $map[$defs['_attributes']['identifier']] = $defs['name']; - } - } - }//end if - - // OPTIMIZATION: Cache the result for subsequent calls during the same import. - $this->propDefMapCache = $map; - - return $map; - }//end extractPropertyDefinitionMap() - - /** - * Transform ArchiMate XML data to objects array in batch (OpenRegister pattern) - * - * This method follows the same pattern as OpenRegister CSV import: - * - Parse ALL sections at once - * - Create objects directly without intermediate normalization - * - Use cached configuration values - * - Minimize object copying and complex transformations - * - * @param array $xmlData Parsed XML data - * @param string $modelIdentifier Model identifier - * - * @return array Array of objects ready for saveObjects() - */ - private function transformArchiMateXmlToObjectsBatch(array $xmlData, string $modelIdentifier): array { - $allObjects = []; - - // Extract propertyDefinitionMap once for all objects. - $propDefMap = $this->extractPropertyDefinitionMap(data: $xmlData); - - // Create model object first. - if (isset($xmlData['_attributes']) === true || isset($xmlData['name']) === true) { - $modelMetadata = [ - 'identifier' => $modelIdentifier, - 'name' => $xmlData['name'] ?? '', - 'documentation' => $xmlData['documentation'] ?? '', - 'properties' => $xmlData['properties'] ?? [], - 'propertyDefinitionMap' => $propDefMap, - ]; - - if (isset($xmlData['_attributes']) === true) { - $modelMetadata = array_merge($modelMetadata, $xmlData['_attributes']); - } - - $allObjects[] = $this->createModelObjectDirect(metadata: $modelMetadata, modelIdentifier: $modelIdentifier); - } - - // Process each section type directly (no intermediate normalization). - $sections = [ - 'elements' => 'element', - 'relationships' => 'relationship', - 'organizations' => 'organization', - 'views' => 'view', - 'property_definitions' => 'property_definition', - ]; - - foreach ($sections as $sectionName => $schemaType) { - $sectionData = $this->findSectionData(xmlData: $xmlData, sectionName: $sectionName); - if (empty($sectionData) === false) { - $sectionObjects = $this->transformSectionObjectsBatch( - sectionData: $sectionData, - schemaType: $schemaType, - modelIdentifier: $modelIdentifier, - propDefMap: $propDefMap - ); - $allObjects = array_merge($allObjects, $sectionObjects); - } - } - - return $allObjects; - }//end transformArchiMateXmlToObjectsBatch() - - /** - * Create model object directly with cached configuration - * - * @param array $metadata Model metadata - * @param string $modelIdentifier Model identifier - * - * @return array Model object with @self structure - */ - private function createModelObjectDirect(array $metadata, string $modelIdentifier): array { - return [ - '@self' => [ - 'register' => $this->cachedConfig['registerId'], - 'schema' => $this->cachedConfig['schemaIds']['model'], - 'id' => $modelIdentifier, - 'published' => date('Y-m-d\TH:i:s\Z'), - ], - 'identifier' => $modelIdentifier, - 'section' => 'model', - 'model_identifier' => $modelIdentifier, - ] + $metadata; - }//end createModelObjectDirect() - - /** - * Find section data efficiently without complex nested searches - * - * @param array $xmlData Parsed XML data - * @param string $sectionName Section name to find - * - * @return array Section data or empty array - */ - private function findSectionData(array $xmlData, string $sectionName): array { - // Direct lookup first. - if (isset($xmlData[$sectionName]) === true) { - return $xmlData[$sectionName]; - } - - // Alternative names lookup. - $alternatives = [ - 'views' => ['diagrams'], - 'organizations' => ['organisation'], - 'property_definitions' => ['propertyDefinitions', 'propertydefinitions'], - ]; - - if (isset($alternatives[$sectionName]) === true) { - foreach ($alternatives[$sectionName] as $altName) { - if (isset($xmlData[$altName]) === true) { - return $xmlData[$altName]; - } - } - } - - return []; - }//end findSectionData() - - /** - * Transform section objects in batch with minimal overhead - * - * @param array $sectionData Section data from XML - * @param string $schemaType Schema type (singular) - * @param string $modelIdentifier Model identifier - * @param array $propDefMap Property definition map - * - * @return array Array of transformed objects - */ - private function transformSectionObjectsBatch( - array $sectionData, - string $schemaType, - string $modelIdentifier, - array $propDefMap, - ): array { - $objects = []; - - // Find items in section (simplified version). - $items = $this->findItemsSimplified(sectionData: $sectionData, sectionType: $schemaType); - - foreach ($items as $item) { - if (is_array($item) === false) { - continue; - } - - $identifier = $this->extractIdentifier(item: $item, sectionName: $schemaType); - if ($identifier === null) { - continue; - } + // Extract pagination parameters. + $limit = $query['limit'] ?? 1000; + // Default limit for large datasets. + $offset = $query['offset'] ?? 0; + $usePagination = $query['use_pagination'] ?? false; + + // Remove pagination parameters from query. + unset($query['limit'], $query['offset'], $query['use_pagination']); - // Create object directly (minimal processing). - $object = [ + // Build base query for register and schema. + $baseQuery = [ '@self' => [ - 'register' => $this->cachedConfig['registerId'], - 'schema' => $this->cachedConfig['schemaIds'][$schemaType], - 'id' => $identifier, - 'published' => date('Y-m-d\TH:i:s\Z'), + 'register' => (int)$registerId, + 'schema' => (int)$schemaId, ], - 'identifier' => $identifier, - 'section' => $schemaType, - 'model_identifier' => $modelIdentifier, - 'xml' => $this->extractEssentialXmlData(item: $item), - // OPTIMIZATION: Store only essential XML data. ]; - // Extract name from XML if it exists. - if (isset($item['name']) === true) { - if (is_array($item['name']) === true && isset($item['name']['_value']) === true) { - $object['name'] = $item['name']['_value']; - } elseif (is_string($item['name']) === true) { - $object['name'] = $item['name']; - } + // Merge with provided query. + $finalQuery = array_merge_recursive($baseQuery, $query); + + // Add pagination if requested. + if ($usePagination !== false && $limit > 0) { + $finalQuery['@pagination'] = [ + 'limit' => (int)$limit, + 'offset' => (int)$offset, + ]; } - // Extract documentation from XML if it exists and set to summary. - if (isset($item['documentation']) === true) { - if (is_array($item['documentation']) === true && isset($item['documentation']['_value']) === true) { - $object['omschrijving'] = $item['documentation']['_value']; - } elseif (is_string($item['documentation']) === true) { - $object['omschrijving'] = $item['documentation']; - } + // Always bound the actual query: OpenRegister's searchObjects() + // only reads the top-level `_limit`/`_offset` keys (see + // MagicMapper/MagicSearchHandler) — `@pagination` above is not + // read by the query layer, so this call was unbounded regardless + // of the `usePagination` toggle. `$limit` already defaults to + // 1000 above, so this is a straight bound, not a behavior change. + $finalQuery['_limit'] = (int)$limit; + if ($usePagination !== false) { + $finalQuery['_offset'] = (int)$offset; } - // Flatten properties efficiently (if present). - if (isset($item['properties']['property']) === true && empty($propDefMap) === false) { - $this->flattenPropertiesBatch( - object: $object, - properties: $item['properties']['property'], - propDefMap: $propDefMap - ); + $paginationValue = 'disabled'; + if ($usePagination === true) { + $paginationValue = 'enabled'; } - $objects[] = $object; - }//end foreach + $this->logger->debug( + "ArchiMateService: Retrieving {$schemaType} objects", + [ + 'register' => $registerId, + 'schema' => $schemaId, + 'query' => $finalQuery, + 'pagination' => $paginationValue, + ] + ); - return $objects; - }//end transformSectionObjectsBatch() + // Use searchObjects method for filtering. + $objects = $objectService->searchObjects($finalQuery); - /** - * Simplified item finding for better performance - * - * @param array $sectionData Section data - * @param string $sectionType Section type - * - * @return array Items array - */ - private function findItemsSimplified(array $sectionData, string $sectionType): array { - // Handle views with diagrams structure. - if ($sectionType === 'view' && isset($sectionData['diagrams']['view']) === true) { - $viewData = $sectionData['diagrams']['view']; - if (isset($viewData[0]) === true) { - return $viewData; + $paginationValue = 'disabled'; + if ($usePagination === true) { + $paginationValue = 'enabled'; } - return [$viewData]; - } - - // Try common patterns. - $patterns = [ - // Singular: element, relationship, etc. - $sectionType, - // Plural: elements, relationships, etc. - $sectionType . 's', - // Organizations use 'item'. - 'item', - // Property definitions. - 'propertyDefinition', - ]; - - foreach ($patterns as $pattern) { - if (isset($sectionData[$pattern]) === true) { - $data = $sectionData[$pattern]; - if (is_array($data) === true && isset($data[0]) === true) { - return $data; - } + $this->logger->debug( + "ArchiMateService: Retrieved {$schemaType} objects", + [ + 'register' => $registerId, + 'schema' => $schemaId, + 'count' => count($objects), + 'pagination' => $paginationValue, + ] + ); - return [$data]; - } - } + return $objects; + } catch (\Exception $e) { + $this->logger->error( + "ArchiMateService: Failed to retrieve {$schemaType} objects", + [ + 'error' => $e->getMessage(), + 'trace' => $e->getTraceAsString(), + ] + ); - // Fallback: treat section data as single item. - return [$sectionData]; - }//end findItemsSimplified() + return []; + }//end try + }//end getObjectsWithPagination() /** - * Flatten properties in batch for better performance - * - * @param array $object Object to add properties to (by reference). - * @param array $properties Properties array from XML. - * @param array $propDefMap Property definition map. + * Check if import is in progress * - * @return void + * @return bool True if import is in progress + * @spec openspec/specs/archimate-import/spec.md */ - private function flattenPropertiesBatch(array &$object, array $properties, array $propDefMap): void { - $props = [$properties]; - if (isset($properties[0]) === true) { - $props = $properties; - } - - $processedProperties = []; - - foreach ($props as $prop) { - if (isset($prop['_attributes']['propertyDefinitionRef']) === false) { - continue; - } - - $defRef = $prop['_attributes']['propertyDefinitionRef']; - $value = $prop['value']['_value'] ?? $prop['value'] ?? null; - - if ($value !== null && isset($propDefMap[$defRef]) === true) { - $propertyName = $propDefMap[$defRef]; - $camelCaseName = $this->convertToCamelCase(propertyName: $propertyName); - $object[$camelCaseName] = $value; - - // Store property mapping for reference. - if (isset($object['_propertyMapping']) === false) { - $object['_propertyMapping'] = []; - } - - $object['_propertyMapping'][$camelCaseName] = $propertyName; - - $processedProperties[] = [ - 'original' => $propertyName, - 'camelCase' => $camelCaseName, - 'value' => $value, - ]; - - // Set slug for Object ID property. - if (strtolower($propertyName) === 'object id') { - $object['@self']['slug'] = $value; - } - }//end if - }//end foreach + public function isImportInProgress(): bool { + // For now, return false as we haven't implemented status tracking yet. + return false; + }//end isImportInProgress() - // OPTIMIZATION: Removed debug logging from tight loop for performance. - }//end flattenPropertiesBatch() + /** + * Check if export is in progress + * + * @return bool True if export is in progress + * @spec openspec/specs/archimate-import/spec.md + */ + public function isExportInProgress(): bool { + // For now, return false as we haven't implemented status tracking yet. + return false; + }//end isExportInProgress() /** * Convert property names with spaces to camelCase for better database compatibility @@ -2702,417 +1047,4 @@ public function getPropertyNameMapping(array $propDefMap): array { return $mapping; }//end getPropertyNameMapping() - - /** - * Calculate optimized statistics for performance reporting - * - * Reads the persisted save outcome from {@see self::$lastSaveResult}. - * - * @return array Statistics array - */ - private function calculateOptimizedStatistics(): array { - $statistics = [ - 'omschrijving' => [ - 'total_objects_created' => 0, - 'total_objects_updated' => 0, - 'total_objects_deleted' => 0, - 'total_objects_skipped' => 0, - 'total_errors' => 0, - ], - ]; - - if ($this->lastSaveResult !== null) { - $saveResult = $this->lastSaveResult; - $statistics['omschrijving'] = [ - 'total_objects_created' => count($saveResult['saved'] ?? []), - 'total_objects_updated' => count($saveResult['updated'] ?? []), - 'total_objects_deleted' => 0, - 'total_objects_skipped' => count($saveResult['skipped'] ?? []), - 'total_errors' => count($saveResult['invalid'] ?? []), - ]; - } - - return $statistics; - }//end calculateOptimizedStatistics() - - /** - * Extract GEMMA type from an object using multiple possible property names - * - * This method tries different variations of GEMMA type property names to ensure - * compatibility with different ArchiMate model variations. - * - * @param array $object The object to extract GEMMA type from - * - * @return string|null The GEMMA type value or null if not found - */ - private function extractGemmaType(array $object): ?string { - // Try various possible property names for GEMMA type. - $possiblePropertyNames = [ - 'gemmaType', - // Standard camelCase conversion of "GEMMA Type". - 'gemmatype', - // Lowercase version. - 'GemmaType', - // PascalCase version. - 'GEMMA_Type', - // Underscore version. - 'gemma_type', - // Lowercase underscore version. - 'GEMMAType', - // All caps first word. - 'type', - // Sometimes just "Type" in models. - 'elementType', - // Alternative naming. - 'componentType', - // Another alternative. - ]; - - foreach ($possiblePropertyNames as $propertyName) { - if (isset($object[$propertyName]) === true && empty($object[$propertyName]) === false) { - $value = (string)$object[$propertyName]; - - // Log the first successful match for debugging. - if ($this->gemmaTypePropertyFound === false) { - $this->logger->debug( - 'GEMMA Type property found', - [ - 'property_name' => $propertyName, - 'value' => $value, - 'object_id' => $object['identifier'] ?? 'unknown', - ] - ); - $this->gemmaTypePropertyFound = true; - } - - return $value; - } - } - - // If no direct property found, check _propertyMapping for original property names. - if (isset($object['_propertyMapping']) === true) { - foreach ($object['_propertyMapping'] as $camelCase => $original) { - // Check if the original property name contains "gemma" or "type". - if (stripos($original, 'gemma') !== false && stripos($original, 'type') !== false) { - if (isset($object[$camelCase]) === true && empty($object[$camelCase]) === false) { - $this->logger->debug( - 'GEMMA Type found via property mapping', - [ - 'camel_case_name' => $camelCase, - 'original_name' => $original, - 'value' => $object[$camelCase], - 'object_id' => $object['identifier'] ?? 'unknown', - ] - ); - return (string)$object[$camelCase]; - } - } - } - } - - return null; - }//end extractGemmaType() - - /** - * Process GEMMA Referentiecomponent-Standaard relationships with Verbindingsrol support - * - * This method analyzes all objects to find Referentiecomponenten and Standaarden, - * then uses relationships to link them together based on Verbindingsrol property. - * Each Referentiecomponent gets two properties: - * - 'recommendedStandards' array for standards with Verbindingsrol = "Aanbevolen" - * - 'mandatoryStandards' array for standards with Verbindingsrol = "Verplicht" - * - * @param array $objects All objects from the import - * - * @return array Objects with enhanced Referentiecomponent data - */ - private function processGemmaReferenceComponentStandards(array $objects): array { - $this->logger->info( - 'Processing GEMMA Referentiecomponent-Standaard relationships with optimized single-pass algorithm' - ); - - // OPTIMIZATION: Single-pass processing - collect all data types at once. - $referenceComponents = []; - $standards = []; - $gemmaRelationshipMap = []; - - // Debug: Count objects and property variations. - $elementCount = 0; - $elementsWithGemmaType = 0; - $gemmaTypeVariations = []; - - // PASS 1: Collect Referentiecomponenten and Standaarden, process relationships immediately. - foreach ($objects as $index => $object) { - // Debug: Count elements and GEMMA types. - if (isset($object['section']) === true && $object['section'] === 'element') { - $elementCount++; - - // Check for various possible GEMMA type property names. - $gemmaTypeValue = $this->extractGemmaType(object: $object); - if ($gemmaTypeValue !== null) { - $elementsWithGemmaType++; - - // Track GEMMA type variations for debugging. - if (isset($gemmaTypeVariations[$gemmaTypeValue]) === false) { - $gemmaTypeVariations[$gemmaTypeValue] = 0; - } - - $gemmaTypeVariations[$gemmaTypeValue]++; - - if ($gemmaTypeValue === 'Referentiecomponent') { - $referenceComponents[$object['identifier']] = $index; - } elseif ($gemmaTypeValue === 'Standaard') { - $standards[$object['identifier']] = $index; - } - } - }//end if - - // Process relationships immediately when found (no separate collection needed). - if (isset($object['section']) === true && $object['section'] === 'relationship') { - $this->processRelationshipImmediate( - relationship: $object, - referenceComponents: $referenceComponents, - standards: $standards, - gemmaRelationshipMap: $gemmaRelationshipMap - ); - } - }//end foreach - - // Enhanced debug logging. - $this->logger->info( - 'GEMMA objects processing complete', - [ - 'total_elements' => $elementCount, - 'elements_with_gemma_type' => $elementsWithGemmaType, - 'gemma_type_variations' => $gemmaTypeVariations, - 'referentiecomponenten_count' => count($referenceComponents), - 'standaarden_count' => count($standards), - 'processed_relationships' => count($gemmaRelationshipMap), - ] - ); - - // Additional debugging if no GEMMA types found. - if ($elementsWithGemmaType === 0 && $elementCount > 0) { - $this->logger->warning( - 'No GEMMA types found in any elements', - [ - 'total_elements_processed' => $elementCount, - 'sample_element_keys' => 'Will need to examine individual objects', - ] - ); - } - - // STEP 2: Apply the processed relationship mappings to Referentiecomponenten. - $enhancedCount = 0; - foreach ($gemmaRelationshipMap as $referenceComponentId => $standardsMap) { - if (isset($referenceComponents[$referenceComponentId]) === true) { - $objectIndex = $referenceComponents[$referenceComponentId]; - - // Remove duplicates and add the properties. - $recommendedStandards = array_unique($standardsMap['aanbevolen']); - $mandatoryStandards = array_unique($standardsMap['verplicht']); - - $objects[$objectIndex]['recommendedStandards'] = $recommendedStandards; - $objects[$objectIndex]['mandatoryStandards'] = $mandatoryStandards; - - // Also add combined array for backward compatibility. - $allStandards = array_unique(array_merge($recommendedStandards, $mandatoryStandards)); - $objects[$objectIndex]['standards'] = $allStandards; - - $this->logger->info( - 'Enhanced Referentiecomponent with categorized standaarden', - [ - 'referentiecomponent_id' => $referenceComponentId, - 'referentiecomponent_name' => $objects[$objectIndex]['name'] ?? 'Unknown', - 'aanbevolen_count' => count($recommendedStandards), - 'verplicht_count' => count($mandatoryStandards), - 'aanbevolen_ids' => $recommendedStandards, - 'verplicht_ids' => $mandatoryStandards, - ] - ); - - $enhancedCount++; - }//end if - }//end foreach - - $this->logger->info( - 'GEMMA Referentiecomponent-Standaard processing completed', - [ - 'referentiecomponenten_enhanced' => $enhancedCount, - 'total_referentiecomponenten' => count($referenceComponents), - 'total_relationships_processed' => count($gemmaRelationshipMap), - ] - ); - - return $objects; - }//end processGemmaReferenceComponentStandards() - - /** - * OPTIMIZATION: Process relationship immediately when found (single-pass algorithm) - * - * @param array $relationship The relationship object. - * @param array $referenceComponents Array of Referentiecomponent identifiers. - * @param array $standards Array of Standaard identifiers. - * @param array $gemmaRelationshipMap The relationship map to update (by reference). - * - * @return void - */ - private function processRelationshipImmediate( - array $relationship, - array $referenceComponents, - array $standards, - array &$gemmaRelationshipMap, - ): void { - // Get source and target from relationship XML or flattened properties. - $source = $this->extractRelationshipEndpoint(relationship: $relationship, endpoint: 'source'); - $target = $this->extractRelationshipEndpoint(relationship: $relationship, endpoint: 'target'); - - if ($source === null || $target === false) { - return; - } - - // Get Verbindingsrol from flattened properties (camelCase: verbindingsrol). - $verbindingsrol = $relationship['verbindingsrol'] ?? null; - - // Skip if no Verbindingsrol is defined. - if ($verbindingsrol === null) { - return; - } - - // Check if one end is a Referentiecomponent and the other is a Standaard. - $refCompId = null; - $standardId = null; - - if (isset($referenceComponents[$source]) === true && isset($standards[$target]) === true) { - // Referentiecomponent -> Standaard. - $refCompId = $source; - $standardId = $target; - } elseif (isset($standards[$source]) === true && isset($referenceComponents[$target]) === true) { - // Standaard -> Referentiecomponent (reverse direction). - $refCompId = $target; - $standardId = $source; - } - - if ($standardId === true) { - // Initialize arrays if not exists. - if (isset($gemmaRelationshipMap[$refCompId]) === false) { - $gemmaRelationshipMap[$refCompId] = [ - 'aanbevolen' => [], - 'verplicht' => [], - ]; - } - - // Add to appropriate array based on Verbindingsrol. - if (strtolower($verbindingsrol) === 'aanbevolen') { - $gemmaRelationshipMap[$refCompId]['aanbevolen'][] = $standardId; - } elseif (strtolower($verbindingsrol) === 'verplicht') { - $gemmaRelationshipMap[$refCompId]['verplicht'][] = $standardId; - } - } - }//end processRelationshipImmediate() - - /** - * Extract relationship endpoint (source or target) from relationship object - * - * @param array $relationship The relationship object - * @param string $endpoint Either 'source' or 'target' - * - * @return string|null The endpoint identifier or null if not found - */ - private function extractRelationshipEndpoint(array $relationship, string $endpoint): ?string { - // Try flattened camelCase property first. - if (isset($relationship[$endpoint]) === true) { - return $relationship[$endpoint]; - } - - // Try XML structure. - if (isset($relationship['xml'][$endpoint]) === true) { - $endpointData = $relationship['xml'][$endpoint]; - - // Handle different XML structures. - if (is_string($endpointData) === true) { - return $endpointData; - } - - if (is_array($endpointData) === true) { - // Try _attributes.href or _value. - if (isset($endpointData['_attributes']['href']) === true) { - return $endpointData['_attributes']['href']; - } - - if (isset($endpointData['_value']) === true) { - return $endpointData['_value']; - } - } - } - - // Try direct XML access for ArchiMate format. - if (isset($relationship['xml']['_attributes']) === true) { - $attr = $relationship['xml']['_attributes']; - if ($endpoint === 'source' && isset($attr['source']) === true) { - return $attr['source']; - } - - if ($endpoint === 'target' && isset($attr['target']) === true) { - return $attr['target']; - } - } - - return null; - }//end extractRelationshipEndpoint() - - /** - * OPTIMIZATION: Extract only essential XML data to reduce memory usage by 20-30% - * - * Instead of storing the complete XML structure, this method extracts only - * the essential data needed for round-trip fidelity and export functionality. - * - * @param array $item The complete XML item data - * - * @return array Essential XML data for storage - */ - private function extractEssentialXmlData(array $item): array { - $essential = []; - - // Always preserve core attributes (needed for export). - if (isset($item['_attributes']) === true) { - $essential['_attributes'] = $item['_attributes']; - } - - // Preserve name and documentation (already extracted to root level but needed for export). - if (isset($item['name']) === true) { - $essential['name'] = $item['name']; - } - - if (isset($item['documentation']) === true) { - $essential['documentation'] = $item['documentation']; - } - - // Preserve properties structure (needed for property mapping). - if (isset($item['properties']) === true) { - $essential['properties'] = $item['properties']; - } - - // For relationships, preserve source/target information. - if (isset($item['source']) === true) { - $essential['source'] = $item['source']; - } - - if (isset($item['target']) === true) { - $essential['target'] = $item['target']; - } - - // Preserve any other critical ArchiMate-specific fields. - $criticalFields = ['type', 'viewpoint', 'accessType', 'isDirected']; - foreach ($criticalFields as $field) { - if (isset($item[$field]) === true) { - $essential[$field] = $item[$field]; - } - } - - // Add a marker to indicate this is essential data (for debugging). - $essential['_essential_data'] = true; - - return $essential; - }//end extractEssentialXmlData() }//end class diff --git a/phpstan-baseline.neon b/phpstan-baseline.neon index a0f1beca4..8e1cabaf0 100644 --- a/phpstan-baseline.neon +++ b/phpstan-baseline.neon @@ -80,26 +80,11 @@ parameters: count: 1 path: lib/Service/AangebodenGebruikService.php - - - message: "#^Method OCA\\\\Stackiq\\\\Service\\\\AangebodenGebruikService\\:\\:getAllObjectsForSchema\\(\\) is unused\\.$#" - count: 1 - path: lib/Service/AangebodenGebruikService.php - - message: "#^Method OCA\\\\Stackiq\\\\Service\\\\AangebodenGebruikService\\:\\:getApplicationsOwnedByOrganisation\\(\\) is unused\\.$#" count: 1 path: lib/Service/AangebodenGebruikService.php - - - message: "#^Method OCA\\\\Stackiq\\\\Service\\\\AangebodenGebruikService\\:\\:getKoppelingenConfiguration\\(\\) is unused\\.$#" - count: 1 - path: lib/Service/AangebodenGebruikService.php - - - - message: "#^Method OCA\\\\Stackiq\\\\Service\\\\AangebodenGebruikService\\:\\:getObjectsRelatedToUuid\\(\\) is unused\\.$#" - count: 1 - path: lib/Service/AangebodenGebruikService.php - - message: "#^Property OCA\\\\Stackiq\\\\Service\\\\AangebodenGebruikService\\:\\:\\$config is never read, only written\\.$#" count: 1 @@ -120,11 +105,6 @@ parameters: count: 2 path: lib/Service/ArchiMateExportService.php - - - message: "#^Method OCA\\\\Stackiq\\\\Service\\\\ArchiMateExportService\\:\\:addDataToXmlNode\\(\\) is unused\\.$#" - count: 1 - path: lib/Service/ArchiMateExportService.php - - message: "#^Result of && is always false\\.$#" count: 1 @@ -155,61 +135,6 @@ parameters: count: 1 path: lib/Service/ArchiMateImportService.php - - - message: "#^Constant OCA\\\\Stackiq\\\\Service\\\\ArchiMateImportService\\:\\:CONFIG_KEYS is unused\\.$#" - count: 1 - path: lib/Service/ArchiMateImportService.php - - - - message: "#^Method OCA\\\\Stackiq\\\\Service\\\\ArchiMateImportService\\:\\:calculateOptimizedStatistics\\(\\) is unused\\.$#" - count: 1 - path: lib/Service/ArchiMateImportService.php - - - - message: "#^Method OCA\\\\Stackiq\\\\Service\\\\ArchiMateImportService\\:\\:debugViewNodeHierarchy\\(\\) is unused\\.$#" - count: 1 - path: lib/Service/ArchiMateImportService.php - - - - message: "#^Method OCA\\\\Stackiq\\\\Service\\\\ArchiMateImportService\\:\\:extractNodesRecursively\\(\\) is unused\\.$#" - count: 1 - path: lib/Service/ArchiMateImportService.php - - - - message: "#^Method OCA\\\\Stackiq\\\\Service\\\\ArchiMateImportService\\:\\:isAssoc\\(\\) is unused\\.$#" - count: 1 - path: lib/Service/ArchiMateImportService.php - - - - message: "#^Method OCA\\\\Stackiq\\\\Service\\\\ArchiMateImportService\\:\\:logMemoryUsage\\(\\) is unused\\.$#" - count: 1 - path: lib/Service/ArchiMateImportService.php - - - - message: "#^Method OCA\\\\Stackiq\\\\Service\\\\ArchiMateImportService\\:\\:saveObjectsDirectToService\\(\\) is unused\\.$#" - count: 1 - path: lib/Service/ArchiMateImportService.php - - - - message: "#^Method OCA\\\\Stackiq\\\\Service\\\\ArchiMateImportService\\:\\:saveObjectsInParallelBatches\\(\\) is unused\\.$#" - count: 1 - path: lib/Service/ArchiMateImportService.php - - - - message: "#^Method OCA\\\\Stackiq\\\\Service\\\\ArchiMateImportService\\:\\:saveObjectsInSingleBatch\\(\\) is unused\\.$#" - count: 1 - path: lib/Service/ArchiMateImportService.php - - - - message: "#^Method OCA\\\\Stackiq\\\\Service\\\\ArchiMateImportService\\:\\:transformSectionObjectsBatch\\(\\) is unused\\.$#" - count: 1 - path: lib/Service/ArchiMateImportService.php - - - - message: "#^Method OCA\\\\Stackiq\\\\Service\\\\ArchiMateImportService\\:\\:transformViewsOptimized\\(\\) is unused\\.$#" - count: 1 - path: lib/Service/ArchiMateImportService.php - - message: "#^Negated boolean expression is always false\\.$#" count: 4 @@ -290,66 +215,11 @@ parameters: count: 2 path: lib/Service/ArchiMateImportService.php - - - message: "#^Constant OCA\\\\Stackiq\\\\Service\\\\ArchiMateService\\:\\:CONFIG_KEYS is unused\\.$#" - count: 1 - path: lib/Service/ArchiMateService.php - - - - message: "#^Method OCA\\\\Stackiq\\\\Service\\\\ArchiMateService\\:\\:calculateObjectStatistics\\(\\) is unused\\.$#" - count: 1 - path: lib/Service/ArchiMateService.php - - - - message: "#^Method OCA\\\\Stackiq\\\\Service\\\\ArchiMateService\\:\\:calculateOptimizedStatistics\\(\\) is unused\\.$#" - count: 1 - path: lib/Service/ArchiMateService.php - - - - message: "#^Method OCA\\\\Stackiq\\\\Service\\\\ArchiMateService\\:\\:convertToOpenRegisterObjects\\(\\) is unused\\.$#" - count: 1 - path: lib/Service/ArchiMateService.php - - - - message: "#^Method OCA\\\\Stackiq\\\\Service\\\\ArchiMateService\\:\\:findItemsInSection\\(\\) is unused\\.$#" - count: 1 - path: lib/Service/ArchiMateService.php - - - - message: "#^Method OCA\\\\Stackiq\\\\Service\\\\ArchiMateService\\:\\:getAmefSchemaIdForType\\(\\) is unused\\.$#" - count: 1 - path: lib/Service/ArchiMateService.php - - message: "#^Method OCA\\\\Stackiq\\\\Service\\\\ArchiMateService\\:\\:getVoorzieningenConfig\\(\\) is unused\\.$#" count: 1 path: lib/Service/ArchiMateService.php - - - message: "#^Method OCA\\\\Stackiq\\\\Service\\\\ArchiMateService\\:\\:initializeCache\\(\\) is unused\\.$#" - count: 1 - path: lib/Service/ArchiMateService.php - - - - message: "#^Method OCA\\\\Stackiq\\\\Service\\\\ArchiMateService\\:\\:isOperationInProgress\\(\\) is unused\\.$#" - count: 1 - path: lib/Service/ArchiMateService.php - - - - message: "#^Method OCA\\\\Stackiq\\\\Service\\\\ArchiMateService\\:\\:logMemoryUsage\\(\\) is unused\\.$#" - count: 1 - path: lib/Service/ArchiMateService.php - - - - message: "#^Method OCA\\\\Stackiq\\\\Service\\\\ArchiMateService\\:\\:saveObjectsToDatabase\\(\\) is unused\\.$#" - count: 1 - path: lib/Service/ArchiMateService.php - - - - message: "#^Method OCA\\\\Stackiq\\\\Service\\\\ArchiMateService\\:\\:transformArchiMateXmlToObjectsBatch\\(\\) is unused\\.$#" - count: 1 - path: lib/Service/ArchiMateService.php - - message: "#^Negated boolean expression is always false\\.$#" count: 4