From b911fcbd81d8137e45b049aa654bc42ff3eef27d Mon Sep 17 00:00:00 2001 From: simonredfern Date: Mon, 31 Aug 2026 08:44:50 +0200 Subject: [PATCH 1/4] Top Users and Top Consumers in v7.0.0 --- .../scala/code/api/util/ErrorMessages.scala | 2 + .../code/api/v6_0_0/JSONFactory6.0.0.scala | 1 + .../scala/code/api/v7_0_0/Http4s700.scala | 160 ++++++++++++++- .../code/api/v7_0_0/JSONFactory7.0.0.scala | 37 ++++ .../main/scala/code/metrics/APIMetrics.scala | 16 ++ .../code/metrics/DoobieMetricsQueries.scala | 182 ++++++++++++++++++ .../code/metrics/ElasticsearchMetrics.scala | 4 + .../scala/code/metrics/MappedMetrics.scala | 107 ++++++++++ .../code/api/v7_0_0/TopConsumersTest.scala | 92 +++++++++ .../scala/code/api/v7_0_0/TopUsersTest.scala | 94 +++++++++ 10 files changed, 694 insertions(+), 1 deletion(-) create mode 100644 obp-api/src/test/scala/code/api/v7_0_0/TopConsumersTest.scala create mode 100644 obp-api/src/test/scala/code/api/v7_0_0/TopUsersTest.scala diff --git a/obp-api/src/main/scala/code/api/util/ErrorMessages.scala b/obp-api/src/main/scala/code/api/util/ErrorMessages.scala index d833b1fd14..84b8fc9de8 100644 --- a/obp-api/src/main/scala/code/api/util/ErrorMessages.scala +++ b/obp-api/src/main/scala/code/api/util/ErrorMessages.scala @@ -417,6 +417,8 @@ object ErrorMessages { val CheckbookOrderNotFound = "OBP-30041: CheckbookOrder not found for Account. " val GetTopApisError = "OBP-30042: Could not get the top apis from database. " + val GetTopUsersError = "OBP-30551: Could not get the top users from database. " + val GetTopConsumersError = "OBP-30552: Could not get the top consumers from database. " val GetMetricsTopConsumersError = "OBP-30045: Could not get the top consumers from database. " val GetAggregateMetricsError = "OBP-30043: Could not get the aggregate metrics from database. " diff --git a/obp-api/src/main/scala/code/api/v6_0_0/JSONFactory6.0.0.scala b/obp-api/src/main/scala/code/api/v6_0_0/JSONFactory6.0.0.scala index 52d6152f36..a4961c34c0 100644 --- a/obp-api/src/main/scala/code/api/v6_0_0/JSONFactory6.0.0.scala +++ b/obp-api/src/main/scala/code/api/v6_0_0/JSONFactory6.0.0.scala @@ -451,6 +451,7 @@ case class TopApiJsonV600( case class TopApisJsonV600(top_apis: List[TopApiJsonV600]) + case class MetricJsonV600( user_id: String, url: String, diff --git a/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala b/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala index e11a5bb425..55f4c40f0b 100644 --- a/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala +++ b/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala @@ -8,7 +8,7 @@ import code.api.Constant._ import code.api.ResourceDocs1_4_0.SwaggerDefinitionsJSON._ import code.api.util.APIUtil.{EmptyBody, _} import code.api.util.{APIUtil, ApiRole, CallContext, CustomJsonFormats, Glossary, NewStyle} -import code.api.util.ApiRole.{canAttachOpenCorridorPromise, canConfigureAmqpBankBroker, canGetMessageOutbox, canRetryMessageOutbox, canSettleOpenCorridor, canCreateAccount, canCreateEntitlementAtAnyBank, canCreateEntitlementAtOneBank, canCreateMetricsArchiveRun, canCreateOrganisation, canCreateRoutingScheme, canCreateTestEmail, canCreateUtilityVendResult, canDeleteEntitlementAtAnyBank, canDeleteOrganisation, canDeleteRoutingScheme, canDeleteSchedulerJobLock, canGetAccountAccessTrace, canGetAnyOrganisation, canGetAnyUser, canGetCacheConfig, canGetCacheInfo, canGetCacheNamespaces, canGetConnectorHealth, canGetCustomersAtOneBank, canGetDatabasePoolInfo, canGetMetricsDiagnostics, canGetMigrations, canGetSchedulerJobLocks, canUpdateBankSupportedRoutingScheme, canUpdateOrganisation, canUpdateRoutingScheme, canUpdateSystemView} +import code.api.util.ApiRole.{canAttachOpenCorridorPromise, canConfigureAmqpBankBroker, canGetMessageOutbox, canRetryMessageOutbox, canSettleOpenCorridor, canCreateAccount, canCreateEntitlementAtAnyBank, canCreateEntitlementAtOneBank, canCreateMetricsArchiveRun, canCreateOrganisation, canCreateRoutingScheme, canCreateTestEmail, canCreateUtilityVendResult, canDeleteEntitlementAtAnyBank, canDeleteOrganisation, canDeleteRoutingScheme, canDeleteSchedulerJobLock, canGetAccountAccessTrace, canGetAnyOrganisation, canGetAnyUser, canGetCacheConfig, canGetCacheInfo, canGetCacheNamespaces, canGetConnectorHealth, canGetCustomersAtOneBank, canGetDatabasePoolInfo, canGetMetricsDiagnostics, canGetMigrations, canGetSchedulerJobLocks, canReadMetrics, canUpdateBankSupportedRoutingScheme, canUpdateOrganisation, canUpdateRoutingScheme, canUpdateSystemView} import code.api.util.CommonsEmailWrapper import code.model.dataAccess.{AuthUser, BankAccountCreation, MappedBank, ResourceUser} import code.consent.Consents @@ -1209,6 +1209,164 @@ object Http4s700 { http4sPartialFunction = Some(getMyMetrics) ) + // ─── getTopUsers ────────────────────────────────────────────────────────────── + + val getTopUsers: HttpRoutes[IO] = HttpRoutes.of[IO] { + case req @ GET -> `prefixPath` / "management" / "metrics" / "top-users" => + EndpointHelpers.withUser(req) { (_, cc) => + for { + httpParams <- NewStyle.function.extractHttpParamsFromUrl(req.uri.renderString) + (obpQueryParams, callContext) <- APIUtil.createQueriesByHttpParamsFuture(httpParams, cc.callContext) + topUsers <- APIMetrics.apiMetrics.vend.getTopUsersFuture(obpQueryParams) map { + APIUtil.unboxFullOrFail(_, callContext, GetTopUsersError) + } + } yield JSONFactory700.createTopUsersJsonV700(topUsers) + } + } + + resourceDocs += ResourceDoc( + implementedInApiVersion, + nameOf(getTopUsers), + "GET", + "/management/metrics/top-users", + "Get Top Users", + s"""Get the users behind the API traffic: one row per distinct user with their call count, + |sorted by count descending. + | + |**On-behalf-of aware**: calls made under a Consent (e.g. by an agent or a TPP) are + |attributed to the granting (on-behalf-of) user, resolved via the consent table — not to + |the consent's technical shadow user. Anonymous calls are excluded. For a given window and + |filters the number of distinct users listed here therefore matches the + |`distinct_user_count` field of GET /management/aggregate-metrics. + | + |require CanReadMetrics role + | + |Should be able to filter on the following fields + | + |eg: /management/metrics/top-users?from_date=$DateWithMsExampleString&to_date=$DateWithMsExampleString&limit=50 + | + |1 from_date (defaults to one year ago) eg:from_date=$DateWithMsExampleString + | + |2 to_date (defaults to the current date) eg:to_date=$DateWithMsExampleString + | + |3 consumer_id (if null ignore) + | + |4 user_id (if null ignore) + | + |5 anon (if null ignore) only support two value : true (return where user_id is null) or false (return where user_id is not null) + | + |6 url (if null ignore), note: can not contain '&'. + | + |7 app_name (if null ignore) + | + |8 implemented_by_partial_function (if null ignore) + | + |9 implemented_in_version (if null ignore) + | + |10 verb (if null ignore) + | + |11 correlation_id (if null ignore) + | + |12 limit (defaults to 50) eg: limit=200 + | + """.stripMargin, + EmptyBody, + JSONFactory700.TopUsersJsonV700(List( + JSONFactory700.TopUserJsonV700(1000, "9ca9a7e4-6d02-40e3-a129-0b2bf89de9b1", "felixsmith"), + JSONFactory700.TopUserJsonV700(250, "8ca8a7e4-6d02-48e3-a029-0b2bf89de9f0", "susan.uk.29@example.com") + )), + List( + $AuthenticatedUserIsRequired, + UserHasMissingRoles, + InvalidFilterParameterFormat, + GetTopUsersError, + UnknownError + ), + apiTagMetric :: apiTagUser :: Nil, + Some(canReadMetrics :: Nil), + http4sPartialFunction = Some(getTopUsers) + ) + + // ─── getTopConsumers ────────────────────────────────────────────────────────── + + val getTopConsumers: HttpRoutes[IO] = HttpRoutes.of[IO] { + case req @ GET -> `prefixPath` / "management" / "metrics" / "top-consumers" => + EndpointHelpers.withUser(req) { (_, cc) => + for { + httpParams <- NewStyle.function.extractHttpParamsFromUrl(req.uri.renderString) + (obpQueryParams, callContext) <- APIUtil.createQueriesByHttpParamsFuture(httpParams, cc.callContext) + topConsumers <- APIMetrics.apiMetrics.vend.getTopConsumersByConsumerIdFuture(obpQueryParams) map { + APIUtil.unboxFullOrFail(_, callContext, GetTopConsumersError) + } + } yield JSONFactory700.createTopConsumersJsonV700(topConsumers) + } + } + + resourceDocs += ResourceDoc( + implementedInApiVersion, + nameOf(getTopConsumers), + "GET", + "/management/metrics/top-consumers", + "Get Top Consumers", + s"""Get the Consumers (apps) behind the API traffic: one row per distinct consumer with its + |call count, sorted by count descending. + | + |Unlike the v3.1.0 version — which joins metric rows to consumers by APP NAME, dropping + |calls whose app name no longer matches a consumer and double-counting duplicate names — + |this groups by the consumer id stored on each metric row. For a given window and filters + |the number of distinct consumers listed here therefore matches the + |`distinct_consumer_count` field of GET /management/aggregate-metrics. Calls that carried + |no consumer are excluded. `app_name` and `developer_email` are empty when the consumer + |row no longer exists. + | + |require CanReadMetrics role + | + |Should be able to filter on the following fields + | + |eg: /management/metrics/top-consumers?from_date=$DateWithMsExampleString&to_date=$DateWithMsExampleString&limit=50 + | + |1 from_date (defaults to one year ago) eg:from_date=$DateWithMsExampleString + | + |2 to_date (defaults to the current date) eg:to_date=$DateWithMsExampleString + | + |3 consumer_id (if null ignore) + | + |4 user_id (if null ignore) + | + |5 anon (if null ignore) only support two value : true (return where user_id is null) or false (return where user_id is not null) + | + |6 url (if null ignore), note: can not contain '&'. + | + |7 app_name (if null ignore) + | + |8 implemented_by_partial_function (if null ignore) + | + |9 implemented_in_version (if null ignore) + | + |10 verb (if null ignore) + | + |11 correlation_id (if null ignore) + | + |12 limit (defaults to 50) eg: limit=200 + | + """.stripMargin, + EmptyBody, + JSONFactory700.TopConsumersJsonV700(List( + JSONFactory700.TopConsumerJsonV700(1000, "7uy8a7e4-6d02-40e3-a129-0b2bf89de8uh", "API-EXPLORER", "developer@example.com"), + JSONFactory700.TopConsumerJsonV700(250, "8uy8a7e4-6d02-40e3-a129-0b2bf89de8uh", "API-Manager", "manager@example.com") + )), + List( + $AuthenticatedUserIsRequired, + UserHasMissingRoles, + InvalidFilterParameterFormat, + GetTopConsumersError, + UnknownError + ), + apiTagMetric :: apiTagApi :: Nil, + Some(canReadMetrics :: Nil), + http4sPartialFunction = Some(getTopConsumers) + ) + // ── Trading Endpoints ────────────────────────────────────────────────── // Route: POST /obp/v7.0.0/banks/BANK_ID/accounts/ACCOUNT_ID/views/VIEW_ID/trading/offers diff --git a/obp-api/src/main/scala/code/api/v7_0_0/JSONFactory7.0.0.scala b/obp-api/src/main/scala/code/api/v7_0_0/JSONFactory7.0.0.scala index 671897dafc..e4da06be41 100644 --- a/obp-api/src/main/scala/code/api/v7_0_0/JSONFactory7.0.0.scala +++ b/obp-api/src/main/scala/code/api/v7_0_0/JSONFactory7.0.0.scala @@ -1851,6 +1851,43 @@ object JSONFactory700 extends MdcLoggable with code.api.util.CustomJsonFormats { // "completed" (a run executed — inspect `run.success`) or // "skipped_already_in_progress" (a run was already running, so none was started; // `in_progress` then describes the lock that blocked it). + // ─── Top Consumers (v7.0.0) ─── + // Grouped by the consumer id stored on the metric row (NOT by app name like v3.1.0), so + // for a given window the number of rows matches aggregate-metrics' distinct_consumer_count. + // app_name / developer_email are empty when the consumer row no longer exists. + case class TopConsumerJsonV700( + count: Int, + consumer_id: String, + app_name: String, + developer_email: String + ) + + case class TopConsumersJsonV700(top_consumers: List[TopConsumerJsonV700]) + + def createTopConsumersJsonV700(topConsumers: List[code.metrics.TopConsumer]): TopConsumersJsonV700 = + TopConsumersJsonV700( + topConsumers.map(topConsumer => + TopConsumerJsonV700(topConsumer.count, topConsumer.consumerId, topConsumer.appName, topConsumer.developerEmail) + ) + ) + + // ─── Top Users (v7.0.0) ─── + // One distinct user and their call count. On-behalf-of aware: consent-borne calls are + // attributed to the granting human (resolved via the consent table), so for a given + // window the number of rows matches aggregate-metrics' distinct_user_count. + case class TopUserJsonV700( + count: Int, + user_id: String, + username: String + ) + + case class TopUsersJsonV700(top_users: List[TopUserJsonV700]) + + def createTopUsersJsonV700(topUsers: List[code.metrics.TopUser]): TopUsersJsonV700 = + TopUsersJsonV700( + topUsers.map(topUser => TopUserJsonV700(topUser.count, topUser.userId, topUser.userName)) + ) + case class TriggerMetricsArchiveRunResponseJsonV700( status: String, message: String, diff --git a/obp-api/src/main/scala/code/metrics/APIMetrics.scala b/obp-api/src/main/scala/code/metrics/APIMetrics.scala index 70c321ffb2..c8b26ae5e8 100644 --- a/obp-api/src/main/scala/code/metrics/APIMetrics.scala +++ b/obp-api/src/main/scala/code/metrics/APIMetrics.scala @@ -149,6 +149,13 @@ trait APIMetrics { def getTopConsumersFuture(queryParams: List[OBPQueryParam]): Future[Box[List[TopConsumer]]] + def getTopUsersFuture(queryParams: List[OBPQueryParam]): Future[Box[List[TopUser]]] + + // Like getTopConsumersFuture but grouped by metric.consumerid (v3.1.0's version joins on + // app NAME, which drops unmatched rows and fans out on duplicate names). Row count for a + // window matches AggregateMetrics.distinctConsumerCount by construction. + def getTopConsumersByConsumerIdFuture(queryParams: List[OBPQueryParam]): Future[Box[List[TopConsumer]]] + def bulkDeleteMetrics(): Boolean } @@ -224,4 +231,13 @@ case class TopConsumer( consumerId: String, appName: String, developerEmail: String +) + +// One distinct user and their call count. On-behalf-of aware: consent-borne calls are +// attributed to the granting human via the consent table (see buildTopUsersQuery), so the +// row count for a window matches AggregateMetrics.distinctUserCount. +case class TopUser( + count: Int, + userId: String, + userName: String ) \ No newline at end of file diff --git a/obp-api/src/main/scala/code/metrics/DoobieMetricsQueries.scala b/obp-api/src/main/scala/code/metrics/DoobieMetricsQueries.scala index 5af4d72b3b..b9f4ef39a7 100644 --- a/obp-api/src/main/scala/code/metrics/DoobieMetricsQueries.scala +++ b/obp-api/src/main/scala/code/metrics/DoobieMetricsQueries.scala @@ -186,6 +186,188 @@ object DoobieMetricsQueries { * @param filters Filter options * @return List of TopConsumer sorted by count descending */ + /** + * Get top consumers by API call count, grouped by metric.consumerid. + * + * Unlike getTopConsumers (v3.1.0 semantics), which joins on metric.appname = consumer.name + * — dropping rows whose app name no longer matches a consumer and fanning out on duplicate + * names — this groups on the consumer id stored on the metric row itself, so for a window + * the row count matches AggregateMetrics.distinctConsumerCount by construction. Rows + * without a consumer id ('' / 'null' / NULL) are excluded, mirroring that count. App name + * and developer email are decoration via LEFT JOIN and empty when the consumer row is gone. + * + * buildFilterConditions is NOT reused here: its unqualified `consumerid` fragment would be + * ambiguous against the joined consumer table, so the supported filters are inlined with + * the metric alias. (exclude_* filters are not supported, as everywhere v6+.) + * + * @return List of TopConsumer sorted by count descending + */ + def getTopConsumersByConsumerId( + fromDate: Date, + toDate: Date, + limit: Int, + filters: MetricsQueryFilters + ): List[TopConsumer] = { + val query = buildTopConsumersByConsumerIdQuery(fromDate, toDate, limit, filters) + DoobieUtil.runQuery(query) + } + + private def buildTopConsumersByConsumerIdQuery( + fromDate: Date, + toDate: Date, + limit: Int, + filters: MetricsQueryFilters + ): ConnectionIO[List[TopConsumer]] = { + val fromTs = new java.sql.Timestamp(fromDate.getTime) + val toTs = new java.sql.Timestamp(toDate.getTime) + + val isSqlServer = DBUtil.isSqlServer + + val baseQuery = if (isSqlServer) { + fr""" + SELECT TOP($limit) count(*), m.consumerid, COALESCE(con.name, ''), COALESCE(con.developeremail, '') + FROM metric m + LEFT JOIN consumer con ON con.consumerid = m.consumerid + WHERE m.date_c >= $fromTs + AND m.date_c <= $toTs + AND m.consumerid IS NOT NULL + AND m.consumerid <> '' + AND m.consumerid <> 'null' + """ + } else { + fr""" + SELECT count(*), m.consumerid, COALESCE(con.name, ''), COALESCE(con.developeremail, '') + FROM metric m + LEFT JOIN consumer con ON con.consumerid = m.consumerid + WHERE m.date_c >= $fromTs + AND m.date_c <= $toTs + AND m.consumerid IS NOT NULL + AND m.consumerid <> '' + AND m.consumerid <> 'null' + """ + } + + val conditions = List( + filters.consumerId.map(v => fr"AND m.consumerid = $v"), + filters.userId.map(v => fr"AND m.userid = $v"), + filters.implementedByPartialFunction.map(v => fr"AND m.implementedbypartialfunction = $v"), + filters.implementedInVersion.map(v => fr"AND m.implementedinversion = $v"), + filters.url.map(v => fr"AND m.url = $v"), + filters.appName.map(v => fr"AND m.appname = $v"), + filters.verb.map(v => fr"AND m.verb = $v"), + filters.correlationId.map(v => fr"AND m.correlationid = $v"), + filters.httpStatusCode.map(v => fr"AND m.httpcode = $v"), + filters.anon.flatMap { + case true => Some(fr"AND m.userid = 'null'") + case false => Some(fr"AND m.userid != 'null'") + } + ).flatten.foldLeft(fr"")(_ ++ _) + + val groupAndOrder = fr""" + GROUP BY m.consumerid, con.name, con.developeremail + ORDER BY count(*) DESC + """ + + val limitClause = if (isSqlServer) fr"" else fr"LIMIT $limit" + + val fullQuery = baseQuery ++ conditions ++ groupAndOrder ++ limitClause + + fullQuery.query[(Long, String, String, String)].to[List].map { rows => + rows.map { case (count, consumerId, appName, developerEmail) => + TopConsumer(count.toInt, consumerId, appName, developerEmail) + } + } + } + + /** + * Get top users by API call count for the given time range — WHO is behind the traffic. + * + * On-behalf-of aware: consent-borne rows are attributed to the granting human via the + * consent table (COALESCE(consent.muserid, metric.userid)), mirroring the + * distinct_user_count of the aggregate-metrics query, so the row count for a window + * matches that field. The display name comes from resourceuser — the metric row's own + * username is empty for consent shadow users. Anonymous rows are excluded. + * + * @return List of TopUser sorted by count descending + */ + def getTopUsers( + fromDate: Date, + toDate: Date, + limit: Int, + filters: MetricsQueryFilters + ): List[TopUser] = { + val query = buildTopUsersQuery(fromDate, toDate, limit, filters) + DoobieUtil.runQuery(query) + } + + def getTopUsersFuture( + fromDate: Date, + toDate: Date, + limit: Int, + filters: MetricsQueryFilters + )(implicit ec: ExecutionContext): Future[List[TopUser]] = { + Future { + getTopUsers(fromDate, toDate, limit, filters) + } + } + + private def buildTopUsersQuery( + fromDate: Date, + toDate: Date, + limit: Int, + filters: MetricsQueryFilters + ): ConnectionIO[List[TopUser]] = { + val fromTs = new java.sql.Timestamp(fromDate.getTime) + val toTs = new java.sql.Timestamp(toDate.getTime) + + val isSqlServer = DBUtil.isSqlServer + + // The consent side of the join is unique-indexed on consent_reference_id (no fan-out); + // resourceuser is joined on the RESOLVED id, so a consent's calls surface under the + // granting human's id and name. buildFilterConditions' unqualified columns stay + // unambiguous: the joined tables have no column names in common with metric. + val baseQuery = if (isSqlServer) { + fr""" + SELECT TOP($limit) count(*), COALESCE(c.muserid, m.userid), COALESCE(ru.name_, '') + FROM metric m + LEFT JOIN mappedconsent c ON m.consent_reference_id = c.consent_reference_id + LEFT JOIN resourceuser ru ON ru.userid_ = COALESCE(c.muserid, m.userid) + WHERE m.date_c >= $fromTs + AND m.date_c <= $toTs + AND COALESCE(c.muserid, m.userid) IS NOT NULL + AND COALESCE(c.muserid, m.userid) <> 'null' + """ + } else { + fr""" + SELECT count(*), COALESCE(c.muserid, m.userid), COALESCE(ru.name_, '') + FROM metric m + LEFT JOIN mappedconsent c ON m.consent_reference_id = c.consent_reference_id + LEFT JOIN resourceuser ru ON ru.userid_ = COALESCE(c.muserid, m.userid) + WHERE m.date_c >= $fromTs + AND m.date_c <= $toTs + AND COALESCE(c.muserid, m.userid) IS NOT NULL + AND COALESCE(c.muserid, m.userid) <> 'null' + """ + } + + val conditions = buildFilterConditions(filters, isNewVersion = false) + + val groupAndOrder = fr""" + GROUP BY COALESCE(c.muserid, m.userid), COALESCE(ru.name_, '') + ORDER BY count(*) DESC + """ + + val limitClause = if (isSqlServer) fr"" else fr"LIMIT $limit" + + val fullQuery = baseQuery ++ conditions ++ groupAndOrder ++ limitClause + + fullQuery.query[(Long, String, String)].to[List].map { rows => + rows.map { case (count, userId, userName) => + TopUser(count.toInt, userId, userName) + } + } + } + def getTopConsumers( fromDate: Date, toDate: Date, diff --git a/obp-api/src/main/scala/code/metrics/ElasticsearchMetrics.scala b/obp-api/src/main/scala/code/metrics/ElasticsearchMetrics.scala index a6a5aaefef..fb80d918d3 100644 --- a/obp-api/src/main/scala/code/metrics/ElasticsearchMetrics.scala +++ b/obp-api/src/main/scala/code/metrics/ElasticsearchMetrics.scala @@ -70,6 +70,10 @@ object ElasticsearchMetrics extends APIMetrics { override def getTopConsumersFuture(queryParams: List[OBPQueryParam]): Future[Box[List[TopConsumer]]] = ??? + override def getTopUsersFuture(queryParams: List[OBPQueryParam]): Future[Box[List[TopUser]]] = ??? + + override def getTopConsumersByConsumerIdFuture(queryParams: List[OBPQueryParam]): Future[Box[List[TopConsumer]]] = ??? + override def bulkDeleteMetrics(): Boolean = { MappedMetric.bulkDelete_!!() } diff --git a/obp-api/src/main/scala/code/metrics/MappedMetrics.scala b/obp-api/src/main/scala/code/metrics/MappedMetrics.scala index 39ad61906e..dccc9515da 100644 --- a/obp-api/src/main/scala/code/metrics/MappedMetrics.scala +++ b/obp-api/src/main/scala/code/metrics/MappedMetrics.scala @@ -581,6 +581,113 @@ object MappedMetrics extends APIMetrics with MdcLoggable{ }} }} + // Smart caching applied - uses determineMetricsCacheTTL based on query date range + // Groups by metric.consumerid — see DoobieMetricsQueries.buildTopConsumersByConsumerIdQuery. + override def getTopConsumersByConsumerIdFuture(queryParams: List[OBPQueryParam]): Future[Box[List[TopConsumer]]] = Future{ + /** + * Please note that "var cacheKey = (randomUUID().toString, randomUUID().toString, randomUUID().toString)" + * is just a temporary value field with UUID values in order to prevent any ambiguity. + * The real value will be assigned by Macro during compile time at this line of a code: + * https://github.com/OpenBankProject/scala-macros/blob/master/macros/src/main/scala/com/tesobe/CacheKeyFromArgumentsMacro.scala#L49 + */ + var cacheKey = (randomUUID().toString, randomUUID().toString, randomUUID().toString) + val cacheTTL = determineMetricsCacheTTL(queryParams) + CacheKeyFromArguments.buildCacheKey {Caching.memoizeSyncWithProvider(Some(cacheKey.toString()))(cacheTTL.seconds){ + { + val fromDate = queryParams.collect { case OBPFromDate(value) => value }.headOption + val toDate = queryParams.collect { case OBPToDate(value) => value }.headOption + val consumerId = queryParams.collect { case OBPConsumerId(value) => value }.headOption + val userId = queryParams.collect { case OBPUserId(value) => value }.headOption + val url = queryParams.collect { case OBPUrl(value) => value }.headOption + val appName = queryParams.collect { case OBPAppName(value) => value }.headOption + val implementedByPartialFunction = queryParams.collect { case OBPImplementedByPartialFunction(value) => value }.headOption + val implementedInVersion = queryParams.collect { case OBPImplementedInVersion(value) => value }.headOption + val verb = queryParams.collect { case OBPVerb(value) => value }.headOption + val anon = queryParams.collect { case OBPAnon(value) => value }.headOption + val correlationId = queryParams.collect { case OBPCorrelationId(value) => value }.headOption + val httpStatusCode = queryParams.collect { case OBPHttpStatusCode(value) => value }.headOption + val limit = queryParams.collect { case OBPLimit(value) => value }.headOption.getOrElse(50) + + val filters = MetricsQueryFilters( + consumerId = consumerId, + userId = userId, + url = url, + appName = appName, + implementedByPartialFunction = implementedByPartialFunction, + implementedInVersion = implementedInVersion, + verb = verb, + anon = anon, + correlationId = correlationId, + httpStatusCode = httpStatusCode, + excludeAppNames = None, + excludeUrlPatterns = None, + excludeImplementedByPartialFunctions = None + ) + + val result: Box[List[TopConsumer]] = tryo { + logger.debug(s"getTopConsumersByConsumerIdFuture using Doobie with filters: $filters, limit: $limit") + DoobieMetricsQueries.getTopConsumersByConsumerId(fromDate.get, toDate.get, limit, filters) + } + result + }} + }} + + // Smart caching applied - uses determineMetricsCacheTTL based on query date range + // Groups by the on-behalf-of-resolved user — see DoobieMetricsQueries.buildTopUsersQuery. + override def getTopUsersFuture(queryParams: List[OBPQueryParam]): Future[Box[List[TopUser]]] = Future{ + /** + * Please note that "var cacheKey = (randomUUID().toString, randomUUID().toString, randomUUID().toString)" + * is just a temporary value field with UUID values in order to prevent any ambiguity. + * The real value will be assigned by Macro during compile time at this line of a code: + * https://github.com/OpenBankProject/scala-macros/blob/master/macros/src/main/scala/com/tesobe/CacheKeyFromArgumentsMacro.scala#L49 + */ + var cacheKey = (randomUUID().toString, randomUUID().toString, randomUUID().toString) + val cacheTTL = determineMetricsCacheTTL(queryParams) + CacheKeyFromArguments.buildCacheKey {Caching.memoizeSyncWithProvider(Some(cacheKey.toString()))(cacheTTL.seconds){ + { + val fromDate = queryParams.collect { case OBPFromDate(value) => value }.headOption + val toDate = queryParams.collect { case OBPToDate(value) => value }.headOption + val consumerId = queryParams.collect { case OBPConsumerId(value) => value }.headOption + val userId = queryParams.collect { case OBPUserId(value) => value }.headOption + val url = queryParams.collect { case OBPUrl(value) => value }.headOption + val appName = queryParams.collect { case OBPAppName(value) => value }.headOption + val excludeAppNames: Option[List[String]] = queryParams.collect { case OBPExcludeAppNames(value) => value }.headOption + val implementedByPartialFunction = queryParams.collect { case OBPImplementedByPartialFunction(value) => value }.headOption + val implementedInVersion = queryParams.collect { case OBPImplementedInVersion(value) => value }.headOption + val verb = queryParams.collect { case OBPVerb(value) => value }.headOption + val anon = queryParams.collect { case OBPAnon(value) => value }.headOption + val correlationId = queryParams.collect { case OBPCorrelationId(value) => value }.headOption + val httpStatusCode = queryParams.collect { case OBPHttpStatusCode(value) => value }.headOption + val excludeUrlPatterns = queryParams.collect { case OBPExcludeUrlPatterns(value) => value }.headOption + val excludeImplementedByPartialFunctions = queryParams.collect { case OBPExcludeImplementedByPartialFunctions(value) => value }.headOption + val limit = queryParams.collect { case OBPLimit(value) => value }.headOption.getOrElse(50) + + val filters = MetricsQueryFilters( + consumerId = consumerId, + userId = userId, + url = url, + appName = appName, + implementedByPartialFunction = implementedByPartialFunction, + implementedInVersion = implementedInVersion, + verb = verb, + anon = anon, + correlationId = correlationId, + httpStatusCode = httpStatusCode, + excludeAppNames = excludeAppNames, + excludeUrlPatterns = excludeUrlPatterns, + excludeImplementedByPartialFunctions = excludeImplementedByPartialFunctions + ) + + val result: Box[List[TopUser]] = tryo { + logger.debug(s"getTopUsersFuture using Doobie with filters: $filters, limit: $limit") + val topUsers = DoobieMetricsQueries.getTopUsers(fromDate.get, toDate.get, limit, filters) + logger.debug(s"getTopUsersFuture returned " + topUsers.length + " rows") + topUsers + } + result + }} + }} + // Smart caching applied - uses determineMetricsCacheTTL based on query date range override def getTopConsumersFuture(queryParams: List[OBPQueryParam]): Future[Box[List[TopConsumer]]] = Future { /** diff --git a/obp-api/src/test/scala/code/api/v7_0_0/TopConsumersTest.scala b/obp-api/src/test/scala/code/api/v7_0_0/TopConsumersTest.scala new file mode 100644 index 0000000000..87a23f112d --- /dev/null +++ b/obp-api/src/test/scala/code/api/v7_0_0/TopConsumersTest.scala @@ -0,0 +1,92 @@ +package code.api.v7_0_0 + +import code.api.util.APIUtil.OAuth._ +import code.api.util.ApiRole.CanReadMetrics +import code.api.util.ErrorMessages.{AuthenticatedUserIsRequired, UserHasMissingRoles} +import code.api.v6_0_0.V600ServerSetup +import code.api.v7_0_0.JSONFactory700.TopConsumersJsonV700 +import code.entitlement.Entitlement +import code.metrics.MetricBatchWriter +import com.openbankproject.commons.model.ErrorMessage +import com.openbankproject.commons.util.ApiVersion +import org.scalatest.Tag + +/** + * Tests GET /obp/v7.0.0/management/metrics/top-consumers (grouped by metric.consumerid, + * unlike the app-name-joined v3.1.0 version). + * + * Counting is asserted behind a url filter unique to this suite's traffic, because + * write_metrics is JVM-wide and other suites (or the top-consumers calls themselves) may + * also land rows on the metric table. + */ +class TopConsumersTest extends V600ServerSetup { + + def v7_0_0_Request = baseRequest / "obp" / "v7.0.0" + + object VersionOfApi extends Tag(ApiVersion.v7_0_0.toString) + object ApiEndpoint1 extends Tag("getTopConsumers") + + feature(s"test $ApiEndpoint1 version $VersionOfApi - Unauthorized access") { + scenario("We will call the endpoint without user credentials", ApiEndpoint1, VersionOfApi) { + When("We make a request v7.0.0") + val request = (v7_0_0_Request / "management" / "metrics" / "top-consumers").GET + val response = makeGetRequest(request) + Then("We should get a 401") + response.code should equal(401) + response.body.extract[ErrorMessage].message should equal(AuthenticatedUserIsRequired) + } + } + + feature(s"test $ApiEndpoint1 version $VersionOfApi - Missing role") { + scenario("We will call the endpoint with user credentials but without a proper entitlement", ApiEndpoint1, VersionOfApi) { + When("We make a request v7.0.0") + val request = (v7_0_0_Request / "management" / "metrics" / "top-consumers").GET <@ (user1) + val response = makeGetRequest(request) + Then("error should be " + UserHasMissingRoles + CanReadMetrics) + response.code should equal(403) + response.body.extract[ErrorMessage].message should be(UserHasMissingRoles + CanReadMetrics) + } + } + + feature(s"test $ApiEndpoint1 version $VersionOfApi - Top consumers by call count") { + scenario("Two consumers make traffic and appear ranked with their counts", ApiEndpoint1, VersionOfApi) { + setPropsValues("write_metrics" -> "true") + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, CanReadMetrics.toString) + + val trafficUrl = "/obp/v5.1.0/banks" + + // 5 calls via testConsumer (user1), 3 via testConsumer2 (user2) — asymmetric on purpose. + val requestBanks1 = (v5_1_0_Request / "banks").GET <@ (user1) + (1 to 5).foreach(_ => makeGetRequest(requestBanks1)) + val requestBanks2 = (v5_1_0_Request / "banks").GET <@ (user2) + (1 to 3).foreach(_ => makeGetRequest(requestBanks2)) + + MetricBatchWriter.flush() + + When("We query top-consumers filtered to the traffic url") + val request = (v7_0_0_Request / "management" / "metrics" / "top-consumers").GET <@ (user1) < "true") + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, CanReadMetrics.toString) + + val trafficUrl = "/obp/v5.1.0/banks" + + // 5 calls as user1, 3 as user2 — asymmetric so a wrong grouping cannot look right. + val requestBanks1 = (v5_1_0_Request / "banks").GET <@ (user1) + (1 to 5).foreach(_ => makeGetRequest(requestBanks1)) + val requestBanks2 = (v5_1_0_Request / "banks").GET <@ (user2) + (1 to 3).foreach(_ => makeGetRequest(requestBanks2)) + + MetricBatchWriter.flush() + + When("We query top-users filtered to the traffic url") + val request = (v7_0_0_Request / "management" / "metrics" / "top-users").GET <@ (user1) < Date: Mon, 31 Aug 2026 09:26:59 +0200 Subject: [PATCH 2/4] Metrics indexes (for activity dashboards) and related migration --- .../code/api/util/migration/Migration.scala | 16 ++ .../MigrationOfActivityDashboardIndexes.scala | 150 ++++++++++++++++++ .../scala/code/metrics/MappedMetrics.scala | 9 +- .../code/model/dataAccess/ResourceUser.scala | 7 +- 4 files changed, 180 insertions(+), 2 deletions(-) create mode 100644 obp-api/src/main/scala/code/api/util/migration/MigrationOfActivityDashboardIndexes.scala diff --git a/obp-api/src/main/scala/code/api/util/migration/Migration.scala b/obp-api/src/main/scala/code/api/util/migration/Migration.scala index ed8fefb1fa..9429b99101 100644 --- a/obp-api/src/main/scala/code/api/util/migration/Migration.scala +++ b/obp-api/src/main/scala/code/api/util/migration/Migration.scala @@ -141,6 +141,8 @@ object Migration extends MdcLoggable { renameCustomerRoleNames() addUniqueIndexOnResourceUserUserId() addIndexOnMappedMetricUserId() + addCompositeIndexOnMetricUserIdDate() + addIndexOnResourceUserCreatedByConsentId() alterRoleNameLength() alterConsentRequestColumnConsumerIdLength() alterMappedConsentColumnConsumerIdLength() @@ -681,6 +683,20 @@ object Migration extends MdcLoggable { MigrationOfUserIdIndexes.addIndexOnMappedMetricUserId(name) } } + + private def addCompositeIndexOnMetricUserIdDate(): Boolean = { + val name = nameOf(addCompositeIndexOnMetricUserIdDate) + runOnce(name) { + MigrationOfActivityDashboardIndexes.addCompositeIndexOnMetricUserIdDate(name) + } + } + + private def addIndexOnResourceUserCreatedByConsentId(): Boolean = { + val name = nameOf(addIndexOnResourceUserCreatedByConsentId) + runOnce(name) { + MigrationOfActivityDashboardIndexes.addIndexOnResourceUserCreatedByConsentId(name) + } + } private def alterRoleNameLength(): Boolean = { val name = nameOf(alterRoleNameLength) diff --git a/obp-api/src/main/scala/code/api/util/migration/MigrationOfActivityDashboardIndexes.scala b/obp-api/src/main/scala/code/api/util/migration/MigrationOfActivityDashboardIndexes.scala new file mode 100644 index 0000000000..8716a35c38 --- /dev/null +++ b/obp-api/src/main/scala/code/api/util/migration/MigrationOfActivityDashboardIndexes.scala @@ -0,0 +1,150 @@ +package code.api.util.migration + +import code.api.util.APIUtil +import code.api.util.migration.Migration.{DbFunction, saveLog} +import code.metrics.MappedMetric +import code.model.dataAccess.ResourceUser +import net.liftweb.common.Full +import net.liftweb.mapper.Schemifier + +/** + * Indexes for the self-service metrics reads behind the Activity dashboards. + * + * Declared on the models too (MappedMetric.dbIndexes / ResourceUser.dbIndexes) so fresh + * deploys get them from Lift's Schemifier; existing databases get them here, under the + * migration framework's control — the composite metric index can take a while to build on + * a large table, which is exactly the kind of DDL ops should schedule, not boot should + * spring on them. + */ +object MigrationOfActivityDashboardIndexes { + + /** + * Composite index on Metric(userid, date_c). + * + * Serves /my/metrics (locks on user ids, filters/sorts/limits on date) and top-users + * (groups by userid over a date range) in one pass: equality on user + range and order + * on date. The single-column metric_userid_idx (MigrationOfUserIdIndexes) still needs a + * per-user sort for these queries and becomes redundant once this exists. + * Note: The table name is "Metric" (capital M) and the date column is "date_c". + */ + def addCompositeIndexOnMetricUserIdDate(name: String): Boolean = { + DbFunction.tableExists(MappedMetric) match { + case true => + val startDate = System.currentTimeMillis() + val commitId: String = APIUtil.gitCommit + var isSuccessful = false + + val executedSql = + DbFunction.maybeWrite(true, Schemifier.infoF _) { + APIUtil.getPropsValue("db.driver") match { + case Full(dbDriver) if dbDriver.contains("com.microsoft.sqlserver.jdbc.SQLServerDriver") => + () => + """ + |-- Check if index exists, if not create it + |IF NOT EXISTS (SELECT 1 FROM sys.indexes WHERE name = 'metric_userid_date_idx' AND object_id = OBJECT_ID('Metric')) + |BEGIN + | CREATE INDEX metric_userid_date_idx ON Metric(userid, date_c); + |END + """.stripMargin + case Full(dbDriver) if dbDriver.contains("com.mysql.cj.jdbc.Driver") => + () => + """ + |-- MySQL: Create index (will fail silently if exists in some versions) + |CREATE INDEX metric_userid_date_idx ON Metric(userid, date_c); + """.stripMargin + case _ => // Default (H2, PostgreSQL, etc.) + () => + """ + |CREATE INDEX IF NOT EXISTS metric_userid_date_idx ON Metric(userid, date_c); + """.stripMargin + } + } + + val endDate = System.currentTimeMillis() + val comment: String = + s"""Added composite index on Metric(userid, date_c) + |Executed SQL: + |$executedSql + |Serves /my/metrics (user ids + date range/order) and top-users (group by userid over a date range). + |Note: Table name is "Metric" (capital M); the date column is "date_c". + |""".stripMargin + isSuccessful = true + saveLog(name, commitId, isSuccessful, startDate, endDate, comment) + isSuccessful + + case false => + val startDate = System.currentTimeMillis() + val commitId: String = APIUtil.gitCommit + val isSuccessful = false + val endDate = System.currentTimeMillis() + val comment: String = + s"""${MappedMetric._dbTableNameLC} table does not exist. Skipping index creation.""".stripMargin + saveLog(name, commitId, isSuccessful, startDate, endDate, comment) + isSuccessful + } + } + + /** + * Index on resourceuser.createdbyconsentid. + * + * The delegation registry: consent-agent fan-down (/my/metrics, /my/banks) and + * CallContext.effectiveHumanUserId look up agent users by the consent that minted them. + * Unindexed this is a full scan of resourceuser on every such request, which matters on + * consent-heavy instances where every consent mints a user row. + */ + def addIndexOnResourceUserCreatedByConsentId(name: String): Boolean = { + DbFunction.tableExists(ResourceUser) match { + case true => + val startDate = System.currentTimeMillis() + val commitId: String = APIUtil.gitCommit + var isSuccessful = false + + val executedSql = + DbFunction.maybeWrite(true, Schemifier.infoF _) { + APIUtil.getPropsValue("db.driver") match { + case Full(dbDriver) if dbDriver.contains("com.microsoft.sqlserver.jdbc.SQLServerDriver") => + () => + """ + |-- Check if index exists, if not create it + |IF NOT EXISTS (SELECT 1 FROM sys.indexes WHERE name = 'resourceuser_createdbyconsentid_idx' AND object_id = OBJECT_ID('resourceuser')) + |BEGIN + | CREATE INDEX resourceuser_createdbyconsentid_idx ON resourceuser(createdbyconsentid); + |END + """.stripMargin + case Full(dbDriver) if dbDriver.contains("com.mysql.cj.jdbc.Driver") => + () => + """ + |-- MySQL: Create index (will fail silently if exists in some versions) + |CREATE INDEX resourceuser_createdbyconsentid_idx ON resourceuser(createdbyconsentid); + """.stripMargin + case _ => // Default (H2, PostgreSQL, etc.) + () => + """ + |CREATE INDEX IF NOT EXISTS resourceuser_createdbyconsentid_idx ON resourceuser(createdbyconsentid); + """.stripMargin + } + } + + val endDate = System.currentTimeMillis() + val comment: String = + s"""Added index on resourceuser.createdbyconsentid + |Executed SQL: + |$executedSql + |Serves the consent-agent delegation fan-down (/my/metrics, /my/banks, effectiveHumanUserId). + |""".stripMargin + isSuccessful = true + saveLog(name, commitId, isSuccessful, startDate, endDate, comment) + isSuccessful + + case false => + val startDate = System.currentTimeMillis() + val commitId: String = APIUtil.gitCommit + val isSuccessful = false + val endDate = System.currentTimeMillis() + val comment: String = + s"""${ResourceUser._dbTableNameLC} table does not exist. Skipping index creation.""".stripMargin + saveLog(name, commitId, isSuccessful, startDate, endDate, comment) + isSuccessful + } + } +} diff --git a/obp-api/src/main/scala/code/metrics/MappedMetrics.scala b/obp-api/src/main/scala/code/metrics/MappedMetrics.scala index dccc9515da..e890e5e2e6 100644 --- a/obp-api/src/main/scala/code/metrics/MappedMetrics.scala +++ b/obp-api/src/main/scala/code/metrics/MappedMetrics.scala @@ -862,7 +862,14 @@ object MappedMetric extends MappedMetric with LongKeyedMetaMapper[MappedMetric] // and new rows are stored in the table "Metric" // - at a fresh sandbox there is no the table "MappedMetric", only "Metric" is present override def dbTableName = "Metric" // define the DB table name - override def dbIndexes = Index(date) :: Index(consumerId) :: Index(consentReferenceId) :: super.dbIndexes + // (userId, date) serves the hot self-service reads — /my/metrics locks on user ids and + // filters/sorts/limits on date, and top-users groups by userid over a date range. The + // composite answers equality-on-user + range-and-order-on-date in one pass, avoiding the + // per-user sort a plain userid index (metric_userid_idx from MigrationOfUserIdIndexes, + // where enabled) still needs; that single-column index becomes redundant once this exists. + // NOTE: on a large existing metric table the Schemifier builds this index at boot — expect + // a one-off slow start on the first deploy that includes it. + override def dbIndexes = Index(date) :: Index(consumerId) :: Index(consentReferenceId) :: Index(userId, date) :: super.dbIndexes } diff --git a/obp-api/src/main/scala/code/model/dataAccess/ResourceUser.scala b/obp-api/src/main/scala/code/model/dataAccess/ResourceUser.scala index 5290dd7487..6697c04976 100644 --- a/obp-api/src/main/scala/code/model/dataAccess/ResourceUser.scala +++ b/obp-api/src/main/scala/code/model/dataAccess/ResourceUser.scala @@ -147,7 +147,12 @@ class ResourceUser extends LongKeyedMapper[ResourceUser] with User with ManyToMa } object ResourceUser extends ResourceUser with LongKeyedMetaMapper[ResourceUser]{ - override def dbIndexes = UniqueIndex(provider_, providerId) ::super.dbIndexes + // userId_ is deliberately NOT declared here: MigrationOfUserIdIndexes creates a stronger + // UNIQUE index on it (resourceuser_userid_unique). CreatedByConsentId is the delegation + // registry — consent-agent fan-down (/my/metrics, /my/banks) and effectiveHumanUserId join + // through it; nothing else indexes it, which matters on consent-heavy instances where every + // consent mints a user row. + override def dbIndexes = UniqueIndex(provider_, providerId) :: Index(CreatedByConsentId) :: super.dbIndexes def getDistinctProviders: List[String] = { /** From a1231e3f5667a48f5fbbd77d49a60050e0063a92 Mon Sep 17 00:00:00 2001 From: simonredfern Date: Mon, 31 Aug 2026 12:43:17 +0200 Subject: [PATCH 3/4] Adding BGv2 to allStaticResourceDocs --- obp-api/src/main/scala/code/api/util/APIUtil.scala | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/obp-api/src/main/scala/code/api/util/APIUtil.scala b/obp-api/src/main/scala/code/api/util/APIUtil.scala index 8f7e9fd004..9e45f49432 100644 --- a/obp-api/src/main/scala/code/api/util/APIUtil.scala +++ b/obp-api/src/main/scala/code/api/util/APIUtil.scala @@ -4874,7 +4874,13 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{ val allowedAnswerTransactionRequestChallengeAttempts = APIUtil.getPropsAsIntValue("answer_transactionRequest_challenge_allowed_attempts").openOr(3) - lazy val allStaticResourceDocs = (code.api.util.http4s.Http4sResourceDocAggregation.v600 + // Base is the v7 aggregation — the newest OBP-standard surface, which already contains the + // v6.0.0-and-older aggregation plus the v7-only endpoints (deduped by URL/method). Basing on + // the v6 aggregation silently excluded v7-only operation ids from everything that resolves + // operation ids through this list (api-collection endpoint validation, top-apis lookups, ...). + // ResourceDocRegistryParityTest pins the invariant that every per-standard surface the + // resource-docs dispatcher can serve is contained here. + lazy val allStaticResourceDocs = (code.api.v7_0_0.Http4s700.allResourceDocs ++ OBP_UKOpenBanking_200.allResourceDocs ++ OBP_UKOpenBanking_310.allResourceDocs ++ OBP_UKOpenBanking_401.allResourceDocs @@ -4885,7 +4891,11 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{ // ++ code.api.MxOF.CNBV9_1_0_0.allResourceDocs // ++ code.api.MxOF.OBP_MXOF_1_0_0.allResourceDocs // ++ code.api.BahrainOBF.v1_0_0.ApiCollector.allResourceDocs - ++ code.api.berlin.group.v1_3.OBP_BERLIN_GROUP_1_3.allResourceDocs).toList + ++ code.api.berlin.group.v1_3.OBP_BERLIN_GROUP_1_3.allResourceDocs + // BGv2 was missing here even though /resource-docs/BGv2 serves it, so a BGv2 operation id + // (e.g. BGv2-getAccountDetails) failed the getAllResourceDocs membership check that + // api-collection-endpoints (and anything else resolving operation ids) relies on. + ++ code.api.berlin.group.v2.Http4sBGv2.resourceDocs).toList def allDynamicResourceDocs= (DynamicEntityHelper.doc ++ DynamicEndpointHelper.doc ++ DynamicEndpoints.dynamicResourceDocs).toList From 60b9f75d17e43fdf8efd3eb90fbcf57f70f3d032 Mon Sep 17 00:00:00 2001 From: simonredfern Date: Mon, 31 Aug 2026 12:43:24 +0200 Subject: [PATCH 4/4] Create ResourceDocRegistryParityTest.scala --- .../util/ResourceDocRegistryParityTest.scala | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 obp-api/src/test/scala/code/api/util/ResourceDocRegistryParityTest.scala diff --git a/obp-api/src/test/scala/code/api/util/ResourceDocRegistryParityTest.scala b/obp-api/src/test/scala/code/api/util/ResourceDocRegistryParityTest.scala new file mode 100644 index 0000000000..520d04a223 --- /dev/null +++ b/obp-api/src/test/scala/code/api/util/ResourceDocRegistryParityTest.scala @@ -0,0 +1,51 @@ +package code.api.util + +import code.setup.ServerSetup +import org.scalatest.Tag + +/** + * Guards the invariant that APIUtil.getAllResourceDocs — the global operation-id + * registry used wherever an operation id must be resolved (api-collection endpoint + * validation, top-apis operation-id lookups, ...) — contains every per-standard + * resource-doc surface the resource-docs dispatcher can serve to API Explorer. + * + * These are two parallel registries (ResourceDocsAPIMethods dispatches per + * standard/version; getAllResourceDocs aggregates them all), and they have drifted + * twice: Berlin Group v2 was served by the dispatcher but missing from the global + * registry (so BGv2-getAccountDetails could not be added to an API collection), + * and the global registry was based on the v6 aggregation, excluding v7-only + * operation ids. When you add a NEW API standard, register its docs in BOTH + * places — and add its surface to this list. + */ +class ResourceDocRegistryParityTest extends ServerSetup { + + object RegistryParityTag extends Tag("ResourceDocRegistryParity") + + private lazy val allOperationIds: Set[String] = + APIUtil.getAllResourceDocs.map(_.operationId).toSet + + private lazy val surfaces: List[(String, Seq[String])] = List( + ("OBP standard (v7 aggregation)", code.api.v7_0_0.Http4s700.allResourceDocs.map(_.operationId).toSeq), + ("Berlin Group v1.3", code.api.berlin.group.v1_3.Http4sBGv13.resourceDocs.map(_.operationId).toSeq), + ("Berlin Group v2", code.api.berlin.group.v2.Http4sBGv2.resourceDocs.map(_.operationId).toSeq), + ("UK Open Banking 2.0.0", code.api.UKOpenBanking.v2_0_0.OBP_UKOpenBanking_200.allResourceDocs.map(_.operationId).toSeq), + ("UK Open Banking 3.1.0", code.api.UKOpenBanking.v3_1_0.OBP_UKOpenBanking_310.allResourceDocs.map(_.operationId).toSeq), + ("UK Open Banking 4.0.1", code.api.UKOpenBanking.v4_0_1.OBP_UKOpenBanking_401.allResourceDocs.map(_.operationId).toSeq) + ) + + feature("getAllResourceDocs contains every per-standard resource-doc surface") { + surfaces.foreach { case (label, operationIds) => + scenario(s"$label operation ids are all resolvable globally", RegistryParityTag) { + operationIds should not be empty + val missing = operationIds.filterNot(allOperationIds.contains) + withClue(s"$label operation ids missing from getAllResourceDocs: ${missing.take(10).mkString(", ")} ") { + missing shouldBe empty + } + } + } + + scenario("the operation id from the sandbox bug report resolves", RegistryParityTag) { + allOperationIds should contain("BGv2-getAccountDetails") + } + } +}