Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Scalability: Improve scaling of service instances #4162

Merged
merged 23 commits into from
Mar 18, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
b7dcdf5
Fix delete org when org contains spaces
richard-cox Mar 10, 2020
b3e03f8
Fix scaling related test scripts
richard-cox Mar 10, 2020
5e23ea7
Fix double fetching of missing user relations for getUser
richard-cox Mar 10, 2020
4d60312
Fix create many services script
richard-cox Mar 11, 2020
7088585
Scaling improvements
richard-cox Mar 11, 2020
1ca5dd4
Fix handling of generic errors to cf endpoints
richard-cox Mar 11, 2020
57f1dff
Fix SI Delete
richard-cox Mar 11, 2020
c62fa1e
Fix UPSI Delete
richard-cox Mar 11, 2020
b04a775
Handle error message being in error.status OR as a string in jetstrea…
richard-cox Mar 11, 2020
25ae44a
Block displaying of service wall and cf routes lists until cf/org/spa…
richard-cox Mar 11, 2020
b999a4f
Scaling Fixes
richard-cox Mar 12, 2020
6845aaa
Fix failing test
richard-cox Mar 12, 2020
9067352
Merge remote-tracking branch 'origin/master' into scale-related-fixes
richard-cox Mar 13, 2020
7e2de04
Fix cf-user getUsers
richard-cox Mar 13, 2020
84d7f21
Merge remote-tracking branch 'origin/scale-related-fixes' into scale-…
richard-cox Mar 13, 2020
c61feef
Fixes after merge
richard-cox Mar 13, 2020
a088e13
Fix unit test
richard-cox Mar 13, 2020
8e45bc7
Merge remote-tracking branch 'origin/master' into scale-related-fixes
richard-cox Mar 16, 2020
d06034d
Merge remote-tracking branch 'origin/scale-related-fixes' into scale-…
richard-cox Mar 16, 2020
15ba660
Fix unit test
richard-cox Mar 17, 2020
e426dc2
Merge remote-tracking branch 'origin/master' into scale-related-fixes…
richard-cox Mar 18, 2020
cf32d00
Fix bug from e2e test
richard-cox Mar 18, 2020
069492c
Remove duplicated line
richard-cox Mar 18, 2020
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
import { HttpRequest } from '@angular/common/http';

import { getActions } from '../../../store/src/actions/action.helper';
import { PaginatedAction } from '../../../store/src/types/pagination.types';
import { ICFAction } from '../../../store/src/types/request.types';
import { CFEntityConfig } from '../cf-types';
import { cfEntityFactory } from '../cf-entity-factory';
import {
applicationEntityType,
Expand All @@ -9,14 +11,14 @@ import {
serviceBindingEntityType,
serviceInstancesEntityType,
} from '../cf-entity-types';
import { CFEntityConfig } from '../cf-types';
import {
createEntityRelationKey,
createEntityRelationPaginationKey,
EntityInlineChildAction,
EntityInlineParentAction,
} from '../entity-relations/entity-relations.types';
import { CFStartAction } from './cf-action.types';
import { HttpRequest } from '@angular/common/http';

export const ASSIGN_ROUTE = '[Application] Assign route';
export const ASSIGN_ROUTE_SUCCESS = '[Application] Assign route success';
Expand Down Expand Up @@ -62,7 +64,7 @@ export class GetAppRoutes extends CFStartAction implements EntityInlineParentAct
parentEntityConfig = new CFEntityConfig(applicationEntityType);
}

export class GetAppServiceBindings extends CFStartAction implements EntityInlineParentAction {
export class GetAppServiceBindings extends CFStartAction implements PaginatedAction, EntityInlineParentAction {
constructor(
public guid: string,
public endpointGuid: string,
Expand All @@ -89,6 +91,7 @@ export class GetAppServiceBindings extends CFStartAction implements EntityInline
entity = [cfEntityFactory(serviceBindingEntityType)];
entityType = serviceBindingEntityType;
options: HttpRequest<any>;
flattenPagination = true;
}

export class AssignRouteToApplication extends CFStartAction
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,12 @@ export class GetServiceInstances
constructor(
public endpointGuid: string,
public paginationKey: string,
public includeRelations: string[] = getServiceInstanceRelations,
public includeRelations: string[] = [
createEntityRelationKey(serviceInstancesEntityType, serviceBindingEntityType),
createEntityRelationKey(serviceInstancesEntityType, servicePlanEntityType),
createEntityRelationKey(servicePlanEntityType, serviceEntityType),
createEntityRelationKey(serviceInstancesEntityType, spaceEntityType),
],
public populateMissing = true
) {
super();
Expand Down
41 changes: 19 additions & 22 deletions src/frontend/packages/cloud-foundry/src/cf-entity-factory.ts
Original file line number Diff line number Diff line change
Expand Up @@ -128,10 +128,9 @@ const ServiceBindingsSchema = new CFServiceBindingEntitySchema({
app: new CFApplicationEntitySchema(),
}, { idAttribute: getAPIResourceGuid })],
service: new CFEntitySchema(serviceEntityType, {}, { idAttribute: getAPIResourceGuid }),
service_plan: new CFEntitySchema(servicePlanEntityType, {}, { idAttribute: getAPIResourceGuid }),
service_plan: ServicePlanSchema,
},
}),
service: ServiceNoPlansSchema
})
}
});
entityCache[serviceBindingEntityType] = ServiceBindingsSchema;
Expand All @@ -154,7 +153,6 @@ const ServiceInstancesSchema = new CFServiceInstanceEntitySchema({
entity: {
service_plan: ServicePlanSchema,
service_bindings: [ServiceBindingsSchema],
service: ServiceSchema
}
});
entityCache[serviceInstancesEntityType] = ServiceInstancesSchema;
Expand Down Expand Up @@ -253,7 +251,6 @@ const ServiceInstancesWithSpaceSchema = new CFServiceInstanceEntitySchema({
service_plan: ServicePlanSchema,
service_bindings: [ServiceBindingsSchema],
space: SpaceSchema.withEmptyDefinition(),
service: ServiceSchema
}
});
entityCache[serviceInstancesWithSpaceEntityType] = ServiceInstancesWithSpaceSchema;
Expand Down Expand Up @@ -328,24 +325,24 @@ const CFUserSchema = new CFUserEntitySchema({
audited_spaces: [createUserOrgSpaceSchema(spaceEntityType, {}, CfUserRoleParams.AUDITED_SPACES)],
}
}, {
idAttribute: getAPIResourceGuid,
processStrategy: (user: APIResource<CfUser>) => {
if (user.entity.username) {
return user;
}
const entity = {
...user.entity,
username: user.metadata.guid
};

return user.metadata ? {
entity,
metadata: user.metadata
} : {
entity
idAttribute: getAPIResourceGuid,
processStrategy: (user: APIResource<CfUser>) => {
if (user.entity.username) {
return user;
}
const entity = {
...user.entity,
username: user.metadata.guid
};
}
});

return user.metadata ? {
entity,
metadata: user.metadata
} : {
entity
};
}
});
entityCache[cfUserEntityType] = CFUserSchema;


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,44 @@ import { CreateRoute, DeleteRoute, GetAllRoutes, NewRoute, UnmapRoute } from '..
import { GetSpaceRoutes } from '../actions/space.actions';
import { CFBasePipelineRequestActionMeta } from '../cf-entity-generator';

export const routesActionBuilders = {
export interface RoutesActionBuilders extends OrchestratedActionBuilders {
create: (id, endpointGuid, route: NewRoute) => CreateRoute;
delete: (
guid,
endpointGuid,
appGuid?: string,
appGuids?: string[],
async?: boolean,
recursive?: boolean
) => DeleteRoute;
getMultiple: (
endpointGuid,
paginationKey: string,
{ includeRelations, populateMissing }: CFBasePipelineRequestActionMeta
) => GetAllRoutes;
unmap: (
guid: string,
appGuid: string,
endpointGuid: string,
clearPaginationKey?: string
) => UnmapRoute;
getAllForApplication: (
applicationGuid: string,
endpointGuid: string,
paginationKey?: string,
includeRelations?: string[]
) => GetAppRoutes;
getAllInSpace: (
spaceGuid: string,
endpointGuid: string,
paginationKey: string,
includeRelations?: string[],
populateMissing?: boolean,
flattenPagination?: boolean
) => GetSpaceRoutes;
}

export const routesActionBuilders: RoutesActionBuilders = {
create: (id, endpointGuid, route: NewRoute) => new CreateRoute(
id,
endpointGuid,
Expand Down Expand Up @@ -59,7 +96,6 @@ export const routesActionBuilders = {
includeRelations?: string[],
populateMissing?: boolean,
flattenPagination?: boolean

) => new GetSpaceRoutes(
spaceGuid,
endpointGuid,
Expand All @@ -68,4 +104,4 @@ export const routesActionBuilders = {
populateMissing,
flattenPagination
)
} as OrchestratedActionBuilders;
};
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,13 @@ import { denormalize } from 'normalizr';
import { Observable, of as observableOf } from 'rxjs';
import { filter, first, map, mergeMap, pairwise, skipWhile, switchMap, withLatestFrom } from 'rxjs/operators';

import { entityCatalog } from '../../../store/src/entity-catalog/entity-catalog.service';
import { isEntityBlocked } from '../../../store/src/entity-service';
import { pathGet } from '../../../core/src/core/utils.service';
import { environment } from '../../../core/src/environments/environment';
import { SetInitialParams } from '../../../store/src/actions/pagination.actions';
import { APIResponse } from '../../../store/src/actions/request.actions';
import { GeneralEntityAppState } from '../../../store/src/app-state';
import { entityCatalog } from '../../../store/src/entity-catalog/entity-catalog.service';
import { isEntityBlocked } from '../../../store/src/entity-service';
import { EntitySchema } from '../../../store/src/helpers/entity-schema';
import { pick } from '../../../store/src/helpers/reducer.helper';
import { RequestInfoState } from '../../../store/src/reducers/api-request-reducer/types';
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,20 +16,18 @@ import { IApp, IRoute } from '../../../../../core/src/core/cf-api.types';
import {
AppMonitorComponentTypes,
} from '../../../../../core/src/shared/components/app-action-monitor-icon/app-action-monitor-icon.component';
import {
DataFunctionDefinition,
} from '../../../../../core/src/shared/components/list/data-sources-controllers/list-data-source';
import { ITableColumn } from '../../../../../core/src/shared/components/list/list-table/table.types';
import { RouterNav } from '../../../../../store/src/actions/router.actions';
import { GeneralEntityAppState } from '../../../../../store/src/app-state';
import { entityCatalog } from '../../../../../store/src/entity-catalog/entity-catalog.service';
import { IEntityMetadata } from '../../../../../store/src/entity-catalog/entity-catalog.types';
import { EntityMonitor } from '../../../../../store/src/monitors/entity-monitor';
import { EntityMonitorFactory } from '../../../../../store/src/monitors/entity-monitor.factory.service';
import { PaginationMonitor } from '../../../../../store/src/monitors/pagination-monitor';
import { PaginationMonitorFactory } from '../../../../../store/src/monitors/pagination-monitor.factory';
import { APIResource } from '../../../../../store/src/types/api.types';
import { PaginatedAction } from '../../../../../store/src/types/pagination.types';
import { CF_ENDPOINT_TYPE } from '../../../cf-types';
import { RoutesActionBuilders } from '../../../entity-action-builders/routes.action-builder';
import {
CfAppRoutesListConfigService,
} from '../../../shared/components/list/list-types/app-route/cf-app-routes-list-config.service';
Expand Down Expand Up @@ -82,7 +80,7 @@ export class ApplicationDeleteComponent<T> {
cellDefinition: {
getValue: (row) => {
const si = isServiceInstance(row.entity.service_instance.entity);
return si ? si.service.entity.label : 'User Service';
return si ? si.service_plan.entity.service.entity.label : 'User Service';
}
},
cellFlex: '2'
Expand All @@ -97,7 +95,7 @@ export class ApplicationDeleteComponent<T> {
type: 'sort',
orderKey: 'creation',
field: 'metadata.created_at'
} as DataFunctionDefinition,
},
cellFlex: '1'
}
];
Expand Down Expand Up @@ -186,7 +184,7 @@ export class ApplicationDeleteComponent<T> {

this.relatedEntities$ = combineLatest(instanceMonitor.currentPage$, routeMonitor.currentPage$).pipe(
filter(([instances, routes]) => !!routes && !!instances),
map(([instances, routes]) => ({ instances, routes }))
map(([instances, routes]) => ({ instances, routes })),
);

// Are we fetching application routes or service instances?
Expand Down Expand Up @@ -240,9 +238,9 @@ export class ApplicationDeleteComponent<T> {
const { appGuid, cfGuid } = this.applicationService;
const instanceAction = AppServiceBindingDataSource.createGetAllServiceBindings(appGuid, cfGuid);

const routeEntity = entityCatalog.getEntity(CF_ENDPOINT_TYPE, routeEntityType);
const routeEntity = entityCatalog.getEntity<IEntityMetadata, null, RoutesActionBuilders>(CF_ENDPOINT_TYPE, routeEntityType);
const actionBuilder = routeEntity.actionOrchestrator.getActionBuilder('getAllForApplication');
const routesAction = actionBuilder(appGuid, cfGuid) as PaginatedAction;
const routesAction = actionBuilder(appGuid, cfGuid);

const instancePaginationKey = instanceAction.paginationKey;
const routesPaginationKey = routesAction.paginationKey;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,15 +15,17 @@ import { CurrentUserPermissionsService } from '../../../../../../core/src/core/c
import { RowState } from '../../../../../../core/src/shared/components/list/data-sources-controllers/list-data-source-types';
import { ListViewTypes } from '../../../../../../core/src/shared/components/list/list.component.types';
import { entityCatalog } from '../../../../../../store/src/entity-catalog/entity-catalog.service';
import { IEntityMetadata } from '../../../../../../store/src/entity-catalog/entity-catalog.types';
import { PaginationMonitorFactory } from '../../../../../../store/src/monitors/pagination-monitor.factory';
import { getPaginationObservables } from '../../../../../../store/src/reducers/pagination-reducer/pagination-reducer.helper';
import { APIResource } from '../../../../../../store/src/types/api.types';
import { CF_ENDPOINT_TYPE } from '../../../../cf-types';
import { ServiceBindingActionBuilders } from '../../../../entity-action-builders/service-binding.action-builders';
import {
AppServiceBindingListConfigService,
} from '../../../../shared/components/list/list-types/app-sevice-bindings/app-service-binding-list-config.service';
import { ServiceActionHelperService } from '../../../../shared/data-services/service-action-helper.service';
import { QParam } from '../../../../shared/q-param';
import { QParam, QParamJoiners } from '../../../../shared/q-param';
import { ApplicationService } from '../../application.service';

@Injectable()
Expand All @@ -33,14 +35,16 @@ export class AppDeleteServiceInstancesListConfigService extends AppServiceBindin
obsCache: { [serviceGuid: string]: Observable<RowState> } = {};

static createFetchServiceBinding = (cfGuid: string, serviceInstanceGuid: string): FetchAllServiceBindings => {
const sgEntity = entityCatalog.getEntity(CF_ENDPOINT_TYPE, serviceBindingEntityType);
const sgEntity = entityCatalog
.getEntity<IEntityMetadata, null, ServiceBindingActionBuilders>(CF_ENDPOINT_TYPE, serviceBindingEntityType);
const actionBuilder = sgEntity.actionOrchestrator.getActionBuilder('getMultiple');
const action = actionBuilder(
cfGuid,
createEntityRelationPaginationKey(serviceEntityType, serviceInstanceGuid)) as FetchAllServiceBindings;
createEntityRelationPaginationKey(serviceEntityType, serviceInstanceGuid) + '-count',
);
action.initialParams['results-per-page'] = 1;
action.initialParams.q = [
new QParam('service_instance_guid', serviceInstanceGuid).toString(),
new QParam('service_instance_guid', serviceInstanceGuid, QParamJoiners.in).toString(),
];
return action;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -60,13 +60,13 @@ export function createGetApplicationAction(guid: string, endpointGuid: string) {
return new GetApplication(
guid,
endpointGuid, [
createEntityRelationKey(applicationEntityType, routeEntityType),
createEntityRelationKey(applicationEntityType, spaceEntityType),
createEntityRelationKey(applicationEntityType, stackEntityType),
createEntityRelationKey(applicationEntityType, serviceBindingEntityType),
createEntityRelationKey(routeEntityType, domainEntityType),
createEntityRelationKey(spaceEntityType, organizationEntityType),
]
createEntityRelationKey(applicationEntityType, routeEntityType),
createEntityRelationKey(applicationEntityType, spaceEntityType),
createEntityRelationKey(applicationEntityType, stackEntityType),
createEntityRelationKey(applicationEntityType, serviceBindingEntityType),
createEntityRelationKey(routeEntityType, domainEntityType),
createEntityRelationKey(spaceEntityType, organizationEntityType),
]
);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,11 @@ import { ActivatedRoute } from '@angular/router';
import { Store } from '@ngrx/store';

import { CFAppState } from '../../../../../cloud-foundry/src/cf-app-state';
import { EntityServiceFactory } from '../../../../../store/src/entity-service-factory.service';
import {
ApplicationStateService,
} from '../../../../../core/src/shared/components/application-state/application-state.service';
import { APP_GUID, CF_GUID, ENTITY_SERVICE } from '../../../../../core/src/shared/entity.tokens';
import { EntityServiceFactory } from '../../../../../store/src/entity-service-factory.service';
import { PaginationMonitorFactory } from '../../../../../store/src/monitors/pagination-monitor.factory';
import { ApplicationService, createGetApplicationAction } from '../application.service';
import { ApplicationEnvVarsHelper } from './application-tabs-base/tabs/build-tab/application-env-vars.service';
Expand All @@ -32,7 +32,7 @@ export function applicationServiceFactory(
);
}

export function entityServiceFactory(
export function cfApplicationEntityServiceFactory(
cfId: string,
id: string,
esf: EntityServiceFactory
Expand Down Expand Up @@ -84,7 +84,7 @@ export function getGuids(type?: string) {
},
{
provide: ENTITY_SERVICE,
useFactory: entityServiceFactory,
useFactory: cfApplicationEntityServiceFactory,
deps: [CF_GUID, APP_GUID, EntityServiceFactory]
},

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ import { NoopAnimationsModule } from '@angular/platform-browser/animations';
import { RouterTestingModule } from '@angular/router/testing';

import { CoreModule } from '../../../../../../../../core/src/core/core.module';
import { EntityServiceFactory } from '../../../../../../../../store/src/entity-service-factory.service';
import { GITHUB_API_URL } from '../../../../../../../../core/src/core/github.helpers';
import {
ApplicationStateService,
Expand All @@ -14,11 +13,12 @@ import { APP_GUID, CF_GUID, ENTITY_SERVICE } from '../../../../../../../../core/
import { SharedModule } from '../../../../../../../../core/src/shared/shared.module';
import { TabNavService } from '../../../../../../../../core/tab-nav.service';
import { ApplicationServiceMock } from '../../../../../../../../core/test-framework/application-service-helper';
import { EntityServiceFactory } from '../../../../../../../../store/src/entity-service-factory.service';
import { AppStoreModule } from '../../../../../../../../store/src/store.module';
import { generateCfStoreModules } from '../../../../../../../test-framework/cloud-foundry-endpoint-service.helper';
import { CloudFoundryComponentsModule } from '../../../../../../shared/components/components.module';
import { ApplicationService } from '../../../../application.service';
import { entityServiceFactory } from '../../../application-base.component';
import { cfApplicationEntityServiceFactory } from '../../../application-base.component';
import { ApplicationPollComponent } from '../../application-poll/application-poll.component';
import { ApplicationPollingService } from '../../application-polling.service';
import { ApplicationEnvVarsHelper } from './application-env-vars.service';
Expand Down Expand Up @@ -57,7 +57,7 @@ describe('BuildTabComponent', () => {
{ provide: APP_GUID, useValue: '' },
{
provide: ENTITY_SERVICE,
useFactory: entityServiceFactory,
useFactory: cfApplicationEntityServiceFactory,
deps: [CF_GUID, APP_GUID, EntityServiceFactory]
},
ApplicationPollingService
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,4 @@ import {
}
]
})
export class ServicesTabComponent {

}
export class ServicesTabComponent { }
Loading