Skip to content

Commit

Permalink
Project/Task/Job Analytics (cvat-ai#6371)
Browse files Browse the repository at this point in the history
  • Loading branch information
klakhov authored Jul 25, 2023
1 parent d72e954 commit a79052f
Show file tree
Hide file tree
Showing 83 changed files with 3,995 additions and 193 deletions.
1 change: 1 addition & 0 deletions .github/workflows/black.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ jobs:
cvat-cli/**/*.py
tests/python/**/*.py
cvat/apps/quality_control/**/*.py
cvat/apps/analytics_report/**/*.py
dir_names: true

- name: Run checks
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/isort.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ jobs:
cvat-cli/**/*.py
tests/python/**/*.py
cvat/apps/quality_control/**/*.py
cvat/apps/analytics_report/**/*.py
dir_names: true

- name: Run checks
Expand Down
6 changes: 4 additions & 2 deletions .stylelintrc.json
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
{
"extends": "stylelint-config-standard",
"extends": "stylelint-config-standard-scss",
"rules": {
"indentation": 4,
"value-keyword-case": null,
Expand All @@ -16,7 +16,9 @@
{
"ignoreTypes": ["first-child"]
}
]
],
"scss/comment-no-empty": null,
"scss/at-extend-no-missing-placeholder": null
},
"ignoreFiles": ["**/*.js", "**/*.ts", "**/*.py"]
}
43 changes: 43 additions & 0 deletions .vscode/launch.json
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,26 @@
],
"justMyCode": false,
},
{
"name": "REST API tests: Attach to RQ analytics reports worker",
"type": "python",
"request": "attach",
"connect": {
"host": "127.0.0.1",
"port": 9095
},
"pathMappings": [
{
"localRoot": "${workspaceFolder}",
"remoteRoot": "/home/django/"
},
{
"localRoot": "${workspaceFolder}/.env",
"remoteRoot": "/opt/venv",
}
],
"justMyCode": false,
},
{
"type": "pwa-chrome",
"request": "launch",
Expand Down Expand Up @@ -240,6 +260,28 @@
},
"console": "internalConsole"
},
{
"name": "server: RQ - analytics reports",
"type": "python",
"request": "launch",
"stopOnEntry": false,
"justMyCode": false,
"python": "${command:python.interpreterPath}",
"program": "${workspaceRoot}/manage.py",
"args": [
"rqworker",
"analytics_reports",
"--worker-class",
"cvat.rqworker.SimpleWorker",
],
"django": true,
"cwd": "${workspaceFolder}",
"env": {
"DJANGO_LOG_SERVER_HOST": "localhost",
"DJANGO_LOG_SERVER_PORT": "8282"
},
"console": "internalConsole"
},
{
"name": "server: RQ - scheduler",
"type": "python",
Expand Down Expand Up @@ -499,6 +541,7 @@
"server: RQ - webhooks",
"server: RQ - scheduler",
"server: RQ - quality reports",
"server: RQ - analytics reports",
"server: RQ - cleaning",
"server: git",
]
Expand Down
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
(<https://github.com/opencv/cvat/pull/6474>)
- \[SDK\] `cvat_sdk.datasets`, a framework-agnostic equivalent of `cvat_sdk.pytorch`
(<https://github.com/opencv/cvat/pull/6428>)
- Analytics for Jobs, Tasks and Projects (<https://github.com/opencv/cvat/pull/6371>)

### Changed
- TDB
Expand Down
2 changes: 1 addition & 1 deletion cvat-canvas/src/typescript/shared.ts
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ export interface DrawnState {
occluded?: boolean;
hidden?: boolean;
lock: boolean;
source: 'AUTO' | 'SEMI-AUTO' | 'MANUAL';
source: 'AUTO' | 'SEMI-AUTO' | 'MANUAL' | 'FILE';
shapeType: string;
points?: number[];
rotation: number;
Expand Down
2 changes: 1 addition & 1 deletion cvat-core/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "cvat-core",
"version": "9.2.1",
"version": "9.3.0",
"description": "Part of Computer Vision Tool which presents an interface for client-side integration",
"main": "src/api.ts",
"scripts": {
Expand Down
183 changes: 183 additions & 0 deletions cvat-core/src/analytics-report.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,183 @@
// Copyright (C) 2023 CVAT.ai Corporation
//
// SPDX-License-Identifier: MIT

import { ArgumentError } from './exceptions';

export interface SerializedDataEntry {
date?: string;
value?: number | Record<string, number>
}

export interface SerializedTransformBinaryOp {
left: string;
operator: string;
right: string;
}

export interface SerializedTransformationEntry {
name: string;
binary?: SerializedTransformBinaryOp;
}

export interface SerializedAnalyticsEntry {
name?: string;
title?: string;
description?: string;
granularity?: string;
default_view?: string;
data_series?: Record<string, SerializedDataEntry[]>;
transformations?: SerializedTransformationEntry[];
}

export interface SerializedAnalyticsReport {
id?: number;
target?: string;
created_date?: string;
statistics?: SerializedAnalyticsEntry[];
}

export enum AnalyticsReportTarget {
JOB = 'job',
TASK = 'task',
PROJECT = 'project',
}

export enum AnalyticsEntryViewType {
HISTOGRAM = 'histogram',
NUMERIC = 'numeric',
}

export class AnalyticsEntry {
#name: string;
#title: string;
#description: string;
#granularity: string;
#defaultView: AnalyticsEntryViewType;
#dataSeries: Record<string, SerializedDataEntry[]>;
#transformations: SerializedTransformationEntry[];

constructor(initialData: SerializedAnalyticsEntry) {
this.#name = initialData.name;
this.#title = initialData.title;
this.#description = initialData.description;
this.#granularity = initialData.granularity;
this.#defaultView = initialData.default_view as AnalyticsEntryViewType;
this.#transformations = initialData.transformations;
this.#dataSeries = this.applyTransformations(initialData.data_series);
}

get name(): string {
return this.#name;
}

get title(): string {
return this.#title;
}

get description(): string {
return this.#description;
}

// Probably need to create enum for this
get granularity(): string {
return this.#granularity;
}

get defaultView(): AnalyticsEntryViewType {
return this.#defaultView;
}

get dataSeries(): Record<string, SerializedDataEntry[]> {
return this.#dataSeries;
}

get transformations(): SerializedTransformationEntry[] {
return this.#transformations;
}

private applyTransformations(
dataSeries: Record<string, SerializedDataEntry[]>,
): Record<string, SerializedDataEntry[]> {
this.#transformations.forEach((transform) => {
if (transform.binary) {
let operator: (left: number, right: number) => number;
switch (transform.binary.operator) {
case '+': {
operator = (left: number, right: number) => left + right;
break;
}
case '-': {
operator = (left: number, right: number) => left - right;
break;
}
case '*': {
operator = (left: number, right: number) => left * right;
break;
}
case '/': {
operator = (left: number, right: number) => (right !== 0 ? left / right : 0);
break;
}
default: {
throw new ArgumentError(
`Cannot apply transformation: got unsupported operator type ${transform.binary.operator}.`,
);
}
}

const leftName = transform.binary.left;
const rightName = transform.binary.right;
dataSeries[transform.name] = dataSeries[leftName].map((left, i) => {
const right = dataSeries[rightName][i];
if (typeof left.value === 'number' && typeof right.value === 'number') {
return {
value: operator(left.value, right.value),
date: left.date,
};
}
return {
value: 0,
date: left.date,
};
});
delete dataSeries[leftName];
delete dataSeries[rightName];
}
});
return dataSeries;
}
}

export default class AnalyticsReport {
#id: number;
#target: AnalyticsReportTarget;
#createdDate: string;
#statistics: AnalyticsEntry[];

constructor(initialData: SerializedAnalyticsReport) {
this.#id = initialData.id;
this.#target = initialData.target as AnalyticsReportTarget;
this.#createdDate = initialData.created_date;
this.#statistics = [];
for (const analyticsEntry of initialData.statistics) {
this.#statistics.push(new AnalyticsEntry(analyticsEntry));
}
}

get id(): number {
return this.#id;
}

get target(): AnalyticsReportTarget {
return this.#target;
}

get createdDate(): string {
return this.#createdDate;
}

get statistics(): AnalyticsEntry[] {
return this.#statistics;
}
}
34 changes: 34 additions & 0 deletions cvat-core/src/api-implementation.ts
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import QualityReport from './quality-report';
import QualityConflict from './quality-conflict';
import QualitySettings from './quality-settings';
import { FramesMetaData } from './frames';
import AnalyticsReport from './analytics-report';

export default function implementAPI(cvat) {
cvat.plugins.list.implementation = PluginRegistry.list;
Expand Down Expand Up @@ -404,5 +405,38 @@ export default function implementAPI(cvat) {
return new FramesMetaData({ ...result });
};

cvat.analytics.performance.reports.implementation = async (filter) => {
checkFilter(filter, {
jobID: isInteger,
taskID: isInteger,
projectID: isInteger,
startDate: isString,
endDate: isString,
});

checkExclusiveFields(filter, ['jobID', 'taskID', 'projectID'], ['startDate', 'endDate']);

const updatedParams: Record<string, string> = {};

if ('taskID' in filter) {
updatedParams.task_id = filter.taskID;
}
if ('jobID' in filter) {
updatedParams.job_id = filter.jobID;
}
if ('projectID' in filter) {
updatedParams.project_id = filter.projectID;
}
if ('startDate' in filter) {
updatedParams.start_date = filter.startDate;
}
if ('endDate' in filter) {
updatedParams.end_date = filter.endDate;
}

const reportData = await serverProxy.analytics.performance.reports(updatedParams);
return new AnalyticsReport(reportData);
};

return cvat;
}
6 changes: 6 additions & 0 deletions cvat-core/src/api.ts
Original file line number Diff line number Diff line change
Expand Up @@ -279,6 +279,12 @@ function build() {
},
},
analytics: {
performance: {
async reports(filter = {}) {
const result = await PluginRegistry.apiWrapper(cvat.analytics.performance.reports, filter);
return result;
},
},
quality: {
async reports(filter: any) {
const result = await PluginRegistry.apiWrapper(cvat.analytics.quality.reports, filter);
Expand Down
1 change: 1 addition & 0 deletions cvat-core/src/enums.ts
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ export enum Source {
MANUAL = 'manual',
SEMI_AUTO = 'semi-auto',
AUTO = 'auto',
FILE = 'file',
}

export enum LogType {
Expand Down
2 changes: 1 addition & 1 deletion cvat-core/src/object-state.ts
Original file line number Diff line number Diff line change
Expand Up @@ -466,7 +466,7 @@ export default class ObjectState {
}),
);

if ([Source.MANUAL, Source.SEMI_AUTO, Source.AUTO].includes(serialized.source)) {
if ([Source.MANUAL, Source.SEMI_AUTO, Source.AUTO, Source.FILE].includes(serialized.source)) {
data.source = serialized.source;
}
if (typeof serialized.zOrder === 'number') {
Expand Down
Loading

0 comments on commit a79052f

Please sign in to comment.