Skip to content

Commit

Permalink
Merge pull request #3212 from cloudfoundry-incubator/scale-latest-apps
Browse files Browse the repository at this point in the history
Scalability: Handle large number of apps in cf dashboards
  • Loading branch information
nwmac authored Jan 9, 2019
2 parents 42dd359 + ccaabed commit 158687b
Show file tree
Hide file tree
Showing 34 changed files with 543 additions and 290 deletions.
42 changes: 40 additions & 2 deletions src/frontend/app/features/cloud-foundry/cf.helpers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,10 @@ import { pathGet } from '../../core/utils.service';
import { SetClientFilter } from '../../store/actions/pagination.actions';
import { RouterNav } from '../../store/actions/router.actions';
import { AppState } from '../../store/app-state';
import { applicationSchemaKey, endpointSchemaKey } from '../../store/helpers/entity-factory';
import { applicationSchemaKey, endpointSchemaKey, entityFactory } from '../../store/helpers/entity-factory';
import { selectPaginationState } from '../../store/selectors/pagination.selectors';
import { APIResource } from '../../store/types/api.types';
import { PaginationEntityState } from '../../store/types/pagination.types';
import { PaginationEntityState, PaginatedAction } from '../../store/types/pagination.types';
import {
CfUser,
CfUserRoleParams,
Expand All @@ -27,7 +27,12 @@ import { ICfRolesState } from '../../store/types/current-user-roles.types';
import { getCurrentUserCFEndpointRolesState } from '../../store/selectors/current-user-roles-permissions-selectors/role.selectors';
import { EndpointModel } from '../../store/types/endpoint.types';
import { selectEntities } from '../../store/selectors/api.selectors';
import { Headers, Http, Request, RequestOptions, URLSearchParams } from '@angular/http';
import { environment } from '../../../environments/environment';
import { getPaginationObservables } from '../../store/reducers/pagination-reducer/pagination-reducer.helper';
import { PaginationMonitorFactory } from '../../shared/monitors/pagination-monitor.factory';

const { proxyAPIVersion, cfAPIVersion } = environment;

export interface IUserRole<T> {
string: string;
Expand Down Expand Up @@ -265,3 +270,36 @@ export function haveMultiConnectedCfs(store: Store<AppState>): Observable<boolea
export function filterEntitiesByGuid<T>(guid: string, array?: Array<APIResource<T>>): Array<APIResource<T>> {
return array ? array.filter(entity => entity.metadata.guid === guid) : null;
}

export function createFetchTotalResultsPagKey(standardActionKey: string): string {
return standardActionKey + '-totalResults';
}

export function fetchTotalResults(
action: PaginatedAction,
store: Store<AppState>,
paginationMonitorFactory: PaginationMonitorFactory
): Observable<number> {

action.paginationKey = createFetchTotalResultsPagKey(action.paginationKey);
action.initialParams['results-per-page'] = 1;
action.flattenPagination = false;

const pagObs = getPaginationObservables({
store,
action,
paginationMonitor: paginationMonitorFactory.create(
action.paginationKey,
entityFactory(action.entityKey)
)
});
// Ensure the request is made by sub'ing to the entities observable
pagObs.entities$.pipe(
first(),
).subscribe();

return pagObs.pagination$.pipe(
filter(pagination => !!pagination && !!pagination.pageRequests && !!pagination.pageRequests[1] && !pagination.pageRequests[1].busy),
map(pag => pag.totalResults)
);
}
Original file line number Diff line number Diff line change
@@ -1,20 +1,20 @@
import { Injectable } from '@angular/core';
import { Store } from '@ngrx/store';
import { Observable } from 'rxjs';
import { first, map, publishReplay, refCount } from 'rxjs/operators';
import { Observable, of as observableOf } from 'rxjs';
import { filter, first, map, publishReplay, refCount, switchMap } from 'rxjs/operators';

import { IApp, ICfV2Info, IOrganization, ISpace } from '../../../core/cf-api.types';
import { EntityService } from '../../../core/entity-service';
import { EntityServiceFactory } from '../../../core/entity-service-factory.service';
import { CfUserService } from '../../../shared/data-services/cf-user.service';
import { PaginationMonitorFactory } from '../../../shared/monitors/pagination-monitor.factory';
import { GetAllApplications } from '../../../store/actions/application.actions';
import { GetCFInfo } from '../../../store/actions/cloud-foundry.actions';
import { FetchAllDomains } from '../../../store/actions/domains.actions';
import { GetAllEndpoints } from '../../../store/actions/endpoint.actions';
import { DeleteOrganization, GetAllOrganizations } from '../../../store/actions/organization.actions';
import { AppState } from '../../../store/app-state';
import {
applicationSchemaKey,
cfInfoSchemaKey,
domainSchemaKey,
endpointSchemaKey,
Expand All @@ -34,8 +34,10 @@ import { getPaginationObservables } from '../../../store/reducers/pagination-red
import { APIResource, EntityInfo } from '../../../store/types/api.types';
import { CfApplicationState } from '../../../store/types/application.types';
import { EndpointModel, EndpointUser } from '../../../store/types/endpoint.types';
import { QParam } from '../../../store/types/pagination.types';
import { CfUser } from '../../../store/types/user.types';
import { ActiveRouteCfOrgSpace } from '../cf-page.types';
import { fetchTotalResults } from '../cf.helpers';

export function appDataSort(app1: APIResource<IApp>, app2: APIResource<IApp>): number {
const app1Date = new Date(app1.metadata.updated_at);
Expand All @@ -57,6 +59,8 @@ export class CloudFoundryEndpointService {
totalMem$: Observable<number>;
paginationSubscription: any;
allApps$: Observable<APIResource<IApp>[]>;
hasAllApps$: Observable<boolean>;
totalApps$: Observable<number>;
users$: Observable<APIResource<CfUser>[]>;
orgs$: Observable<APIResource<IOrganization>[]>;
info$: Observable<EntityInfo<APIResource<ICfV2Info>>>;
Expand All @@ -80,7 +84,6 @@ export class CloudFoundryEndpointService {
createEntityRelationKey(organizationSchemaKey, domainSchemaKey),
createEntityRelationKey(organizationSchemaKey, quotaDefinitionSchemaKey),
createEntityRelationKey(organizationSchemaKey, privateDomainsSchemaKey),
createEntityRelationKey(spaceSchemaKey, applicationSchemaKey),
createEntityRelationKey(spaceSchemaKey, serviceInstancesSchemaKey),
createEntityRelationKey(spaceSchemaKey, routeSchemaKey), // Not really needed at top level, but if we drop down into an org with
// lots of spaces it saves n x routes requests
Expand All @@ -97,12 +100,35 @@ export class CloudFoundryEndpointService {
]);
}

public static fetchAppCount(
store: Store<AppState>,
pmf: PaginationMonitorFactory,
cfGuid: string,
orgGuid?: string,
spaceGuid?: string)
: Observable<number> {
const parentSchemaKey = spaceGuid ? spaceSchemaKey : orgGuid ? organizationSchemaKey : 'cf';
const uniqueKey = spaceGuid || orgGuid || cfGuid;
const action = new GetAllApplications(createEntityRelationPaginationKey(parentSchemaKey, uniqueKey), cfGuid);
action.initialParams = {
q: []
};
if (orgGuid) {
action.initialParams.q.push(new QParam('organization_guid', orgGuid, ' IN '));
}
if (spaceGuid) {
action.initialParams.q.push(new QParam('space_guid', spaceGuid, ' IN '));
}
return fetchTotalResults(action, store, pmf);
}

constructor(
public activeRouteCfOrgSpace: ActiveRouteCfOrgSpace,
private store: Store<AppState>,
private entityServiceFactory: EntityServiceFactory,
private cfUserService: CfUserService,
private paginationMonitorFactory: PaginationMonitorFactory
private paginationMonitorFactory: PaginationMonitorFactory,
private pmf: PaginationMonitorFactory
) {
this.cfGuid = activeRouteCfOrgSpace.cfGuid;
this.getAllOrgsAction = CloudFoundryEndpointService.createGetAllOrganizations(this.cfGuid);
Expand All @@ -127,7 +153,7 @@ export class CloudFoundryEndpointService {

}

constructCoreObservables() {
private constructCoreObservables() {
this.endpoint$ = this.cfEndpointEntityService.waitForEntity$;

this.orgs$ = getPaginationObservables<APIResource<IOrganization>>({
Expand All @@ -143,23 +169,44 @@ export class CloudFoundryEndpointService {

this.info$ = this.cfInfoEntityService.waitForEntity$;

this.allApps$ = this.orgs$.pipe(
map(orgs => [].concat(...orgs.map(org => org.entity.spaces))),
map((spaces: APIResource<ISpace>[]) => [].concat(...spaces.map(space => space ? space.entity.apps : [])))
);
this.constructAppObservables();

this.fetchDomains();
}

constructSecondaryObservable() {
constructAppObservables() {
const action = new GetAllApplications(createEntityRelationPaginationKey('cf', this.cfGuid), this.cfGuid);

const pagObs = getPaginationObservables<APIResource<IApp>>({
store: this.store,
action,
paginationMonitor: this.pmf.create(action.paginationKey, entityFactory(action.entityKey))
});

this.allApps$ = pagObs.entities$.pipe(// Ensure we sub to entities to kick off fetch process
switchMap(() => pagObs.pagination$),
filter(pagination => !!pagination && !!pagination.pageRequests && !!pagination.pageRequests[1] && !pagination.pageRequests[1].busy),
switchMap(pagination => pagination.maxedResults ? observableOf(null) : pagObs.entities$)
);

this.hasAllApps$ = this.allApps$.pipe(
map((allApps: APIResource<IApp>[]) => !!allApps)
);

this.totalApps$ = pagObs.pagination$.pipe(
map(pag => pag.totalResults)
);
}

private constructSecondaryObservable() {

this.hasSSHAccess$ = this.info$.pipe(
map(p => !!(p.entity.entity &&
p.entity.entity.app_ssh_endpoint &&
p.entity.entity.app_ssh_host_key_fingerprint &&
p.entity.entity.app_ssh_oauth_client))
);
this.totalMem$ = this.allApps$.pipe(map(a => this.getMetricFromApps(a, 'memory')));
this.totalMem$ = this.allApps$.pipe(map(apps => this.getMetricFromApps(apps, 'memory')));

this.connected$ = this.endpoint$.pipe(
map(p => p.entity.connectionStatus === 'connected')
Expand All @@ -169,35 +216,29 @@ export class CloudFoundryEndpointService {

}

getAppsInOrg(
public getAppsInOrgViaAllApps(
org: APIResource<IOrganization>
): Observable<APIResource<IApp>[]> {
return this.allApps$.pipe(
filter(allApps => !!allApps),
map(allApps => {
const orgSpaces = org.entity.spaces.map(s => s.metadata.guid);
return allApps.filter(a => orgSpaces.indexOf(a.entity.space_guid) !== -1);
})
);
}

getAppsInSpace(
public getAppsInSpaceViaAllApps(
space: APIResource<ISpace>
): Observable<APIResource<IApp>[]> {
return this.allApps$.pipe(
filter(allApps => !!allApps),
map(apps => {
return apps.filter(a => a.entity.space_guid === space.metadata.guid);
})
);
}

getAggregateStat(
org: APIResource<IOrganization>,
statMetric: string
): Observable<number> {
return this.getAppsInOrg(org).pipe(
map(apps => this.getMetricFromApps(apps, statMetric))
);
}
public getMetricFromApps(
apps: APIResource<IApp>[],
statMetric: string
Expand All @@ -208,7 +249,7 @@ export class CloudFoundryEndpointService {
.reduce((a, t) => a + t, 0) : 0;
}

fetchDomains = () => {
public fetchDomains = () => {
const action = new FetchAllDomains(this.cfGuid);
this.paginationSubscription = getPaginationObservables<APIResource>(
{
Expand All @@ -223,7 +264,7 @@ export class CloudFoundryEndpointService {
).entities$.subscribe();
}

deleteOrg(orgGuid: string, endpointGuid: string) {
public deleteOrg(orgGuid: string, endpointGuid: string) {
this.store.dispatch(new DeleteOrganization(orgGuid, endpointGuid));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ import { GetAllOrgUsers, GetOrganization } from '../../../store/actions/organiza
import { DeleteSpace } from '../../../store/actions/space.actions';
import { AppState } from '../../../store/app-state';
import {
applicationSchemaKey,
cfUserSchemaKey,
domainSchemaKey,
entityFactory,
Expand All @@ -36,6 +35,18 @@ import { ActiveRouteCfOrgSpace } from '../cf-page.types';
import { getOrgRolesString } from '../cf.helpers';
import { CloudFoundryEndpointService } from './cloud-foundry-endpoint.service';

export const createQuotaDefinition = (orgGuid: string): APIResource<IQuotaDefinition> => ({
entity: {
memory_limit: -1,
app_instance_limit: -1,
instance_memory_limit: -1,
name: 'None assigned',
organization_guid: orgGuid,
total_services: -1,
total_routes: -1
},
metadata: null
});

@Injectable()
export class CloudFoundryOrganizationService {
Expand All @@ -50,6 +61,7 @@ export class CloudFoundryOrganizationService {
spaces$: Observable<APIResource<ISpace>[]>;
appInstances$: Observable<number>;
apps$: Observable<APIResource<IApp>[]>;
appCount$: Observable<number>;
org$: Observable<EntityInfo<APIResource<IOrganization>>>;
allOrgUsers$: Observable<APIResource<CfUser>[]>;
usersPaginationKey: string;
Expand All @@ -60,8 +72,7 @@ export class CloudFoundryOrganizationService {
private entityServiceFactory: EntityServiceFactory,
private cfUserService: CfUserService,
private paginationMonitorFactory: PaginationMonitorFactory,
private cfEndpointService: CloudFoundryEndpointService

private cfEndpointService: CloudFoundryEndpointService,
) {
this.orgGuid = activeRouteCfOrgSpace.orgGuid;
this.cfGuid = activeRouteCfOrgSpace.cfGuid;
Expand All @@ -83,7 +94,6 @@ export class CloudFoundryOrganizationService {
createEntityRelationKey(organizationSchemaKey, quotaDefinitionSchemaKey),
createEntityRelationKey(organizationSchemaKey, privateDomainsSchemaKey),
createEntityRelationKey(spaceSchemaKey, serviceInstancesSchemaKey),
createEntityRelationKey(spaceSchemaKey, applicationSchemaKey),
createEntityRelationKey(spaceSchemaKey, routeSchemaKey),
];
if (!isAdmin) {
Expand Down Expand Up @@ -128,13 +138,34 @@ export class CloudFoundryOrganizationService {
}

private initialiseAppObservables() {
this.apps$ = this.spaces$.pipe(this.getFlattenedList('apps'));
this.apps$ = this.org$.pipe(
switchMap(org => this.cfEndpointService.getAppsInOrgViaAllApps(org.entity))
);
this.appInstances$ = this.apps$.pipe(
filter($apps => !!$apps),
map(getStartedAppInstanceCount)
);

this.totalMem$ = this.apps$.pipe(map(a => this.cfEndpointService.getMetricFromApps(a, 'memory')));

this.appCount$ = this.cfEndpointService.hasAllApps$.pipe(
switchMap(hasAllApps => hasAllApps ? this.countExistingApps() : this.fetchAppCount()),
);
}

private countExistingApps(): Observable<number> {
return this.apps$.pipe(
map(apps => apps.length)
);
}

private fetchAppCount(): Observable<number> {
return CloudFoundryEndpointService.fetchAppCount(
this.store,
this.paginationMonitorFactory,
this.activeRouteCfOrgSpace.cfGuid,
this.activeRouteCfOrgSpace.orgGuid
);
}

private initialiseOrgObservables() {
Expand Down
Loading

0 comments on commit 158687b

Please sign in to comment.