-
Notifications
You must be signed in to change notification settings - Fork 1.5k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Loading status checks…
feat(cdp): Masking destinations (#24266)
1 parent
39467be
commit 524ebff
Showing
25 changed files
with
809 additions
and
151 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,130 @@ | ||
| import { exec } from '@posthog/hogvm' | ||
| import { createHash } from 'crypto' | ||
|
|
||
| import { CdpRedis } from './redis' | ||
| import { HogFunctionInvocationGlobals, HogFunctionType } from './types' | ||
|
|
||
| export const BASE_REDIS_KEY = process.env.NODE_ENV == 'test' ? '@posthog-test/hog-masker' : '@posthog/hog-masker' | ||
| const REDIS_KEY_TOKENS = `${BASE_REDIS_KEY}/mask` | ||
|
|
||
| // NOTE: These are controlled via the api so are more of a sanity fallback | ||
| const MASKER_MAX_TTL = 60 * 60 * 24 | ||
| const MASKER_MIN_TTL = 1 | ||
|
|
||
| type MaskContext = { | ||
| hogFunctionId: string | ||
| hash: string | ||
| increment: number | ||
| ttl: number | ||
| allowedExecutions: number | ||
| threshold: number | null | ||
| } | ||
|
|
||
| type HogInvocationContext = { | ||
| globals: HogFunctionInvocationGlobals | ||
| hogFunction: HogFunctionType | ||
| } | ||
|
|
||
| type HogInvocationContextWithMasker = HogInvocationContext & { | ||
| masker?: MaskContext | ||
| } | ||
|
|
||
| /** | ||
| * HogMasker | ||
| * | ||
| * Responsible for determining if a function is "masked" or not based on the function configuration | ||
| */ | ||
|
|
||
| // Hog masker is meant to be done per batch | ||
| export class HogMasker { | ||
| constructor(private redis: CdpRedis) {} | ||
|
|
||
| public async filterByMasking(invocations: HogInvocationContext[]): Promise<{ | ||
| masked: HogInvocationContext[] | ||
| notMasked: HogInvocationContext[] | ||
| }> { | ||
| const invocationsWithMasker: HogInvocationContextWithMasker[] = [...invocations] | ||
| const masks: Record<string, MaskContext> = {} | ||
|
|
||
| // We find all functions that have a mask and we load their masking from redis | ||
| invocationsWithMasker.forEach((item) => { | ||
| if (item.hogFunction.masking) { | ||
| // TODO: Catch errors | ||
| const value = exec(item.hogFunction.masking.bytecode, { | ||
| globals: item.globals, | ||
| timeout: 50, | ||
| maxAsyncSteps: 0, | ||
| }) | ||
| // What to do if it is null.... | ||
| const hash = createHash('md5').update(String(value.result)).digest('hex').substring(0, 32) | ||
| const hashKey = `${item.hogFunction.id}:${hash}` | ||
| masks[hashKey] = masks[hashKey] || { | ||
| hash, | ||
| hogFunctionId: item.hogFunction.id, | ||
| increment: 0, | ||
| ttl: Math.max( | ||
| MASKER_MIN_TTL, | ||
| Math.min(MASKER_MAX_TTL, item.hogFunction.masking.ttl ?? MASKER_MAX_TTL) | ||
| ), | ||
| threshold: item.hogFunction.masking.threshold, | ||
| allowedExecutions: 0, | ||
| } | ||
|
|
||
| masks[hashKey]!.increment++ | ||
| item.masker = masks[hashKey] | ||
| } | ||
| }) | ||
|
|
||
| if (Object.keys(masks).length === 0) { | ||
| return { masked: [], notMasked: invocations } | ||
| } | ||
|
|
||
| const result = await this.redis.usePipeline({ name: 'masker', failOpen: true }, (pipeline) => { | ||
| Object.values(masks).forEach(({ hogFunctionId, hash, increment, ttl }) => { | ||
| pipeline.incrby(`${REDIS_KEY_TOKENS}/${hogFunctionId}/${hash}`, increment) | ||
| // @ts-expect-error - NX is not typed in ioredis | ||
| pipeline.expire(`${REDIS_KEY_TOKENS}/${hogFunctionId}/${hash}`, ttl, 'NX') | ||
| }) | ||
| }) | ||
|
|
||
| Object.values(masks).forEach((masker, index) => { | ||
| const newValue: number | null = result ? result[index * 2][1] : null | ||
| if (newValue === null) { | ||
| // We fail closed here as with a masking config the typical case will be not to send | ||
| return | ||
| } | ||
|
|
||
| const oldValue = newValue - masker.increment | ||
|
|
||
| // Simplest case - the previous value was 0 | ||
| masker.allowedExecutions = oldValue === 0 ? 1 : 0 | ||
|
|
||
| if (masker.threshold) { | ||
| // TRICKY: We minus 1 to account for the "first" execution | ||
| const thresholdsPasses = | ||
| Math.floor((newValue - 1) / masker.threshold) - Math.floor((oldValue - 1) / masker.threshold) | ||
|
|
||
| if (thresholdsPasses) { | ||
| masker.allowedExecutions = thresholdsPasses | ||
| } | ||
| } | ||
| }) | ||
|
|
||
| return invocationsWithMasker.reduce( | ||
| (acc, item) => { | ||
| if (item.masker) { | ||
| if (item.masker.allowedExecutions > 0) { | ||
| item.masker.allowedExecutions-- | ||
| acc.notMasked.push(item) | ||
| } else { | ||
| acc.masked.push(item) | ||
| } | ||
| } else { | ||
| acc.notMasked.push(item) | ||
| } | ||
| return acc | ||
| }, | ||
| { masked: [], notMasked: [] } as { masked: HogInvocationContext[]; notMasked: HogInvocationContext[] } | ||
| ) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,152 @@ | ||
| // NOTE: PostIngestionEvent is our context event - it should never be sent directly to an output, but rather transformed into a lightweight schema | ||
|
|
||
| import { captureException } from '@sentry/node' | ||
| import { createPool } from 'generic-pool' | ||
| import { Pipeline, Redis } from 'ioredis' | ||
|
|
||
| import { PluginsServerConfig } from '../types' | ||
| import { timeoutGuard } from '../utils/db/utils' | ||
| import { status } from '../utils/status' | ||
| import { createRedisClient } from '../utils/utils' | ||
|
|
||
| type WithCheckRateLimit<T> = { | ||
| checkRateLimit: (key: string, now: number, cost: number, poolMax: number, fillRate: number, expiry: number) => T | ||
| } | ||
|
|
||
| export type CdpRedisClientPipeline = Pipeline & WithCheckRateLimit<number> | ||
|
|
||
| export type CdpRedisClient = Omit<Redis, 'pipeline'> & | ||
| WithCheckRateLimit<Promise<number>> & { | ||
| pipeline: () => CdpRedisClientPipeline | ||
| } | ||
|
|
||
| export type CdpRedisOptions = { | ||
| name: string | ||
| timeout?: number | ||
| failOpen?: boolean | ||
| } | ||
|
|
||
| export type CdpRedis = { | ||
| useClient: <T>(options: CdpRedisOptions, callback: (client: CdpRedisClient) => Promise<T>) => Promise<T | null> | ||
| usePipeline: ( | ||
| options: CdpRedisOptions, | ||
| callback: (pipeline: CdpRedisClientPipeline) => void | ||
| ) => Promise<Array<[Error | null, any]> | null> | ||
| } | ||
|
|
||
| // NOTE: We ideally would have this in a file but the current build step doesn't handle anything other than .ts files | ||
| const LUA_TOKEN_BUCKET = ` | ||
| local key = KEYS[1] | ||
| local now = ARGV[1] | ||
| local cost = ARGV[2] | ||
| local poolMax = ARGV[3] | ||
| local fillRate = ARGV[4] | ||
| local expiry = ARGV[5] | ||
| local before = redis.call('hget', key, 'ts') | ||
| -- If we don't have a timestamp then we set it to now and fill up the bucket | ||
| if before == false then | ||
| local ret = poolMax - cost | ||
| redis.call('hset', key, 'ts', now) | ||
| redis.call('hset', key, 'pool', ret) | ||
| redis.call('expire', key, expiry) | ||
| return ret | ||
| end | ||
| -- We update the timestamp if it has changed | ||
| local timeDiffSeconds = now - before | ||
| if timeDiffSeconds > 0 then | ||
| redis.call('hset', key, 'ts', now) | ||
| else | ||
| timeDiffSeconds = 0 | ||
| end | ||
| -- Calculate how much should be refilled in the bucket and add it | ||
| local owedTokens = timeDiffSeconds * fillRate | ||
| local currentTokens = redis.call('hget', key, 'pool') | ||
| if currentTokens == false then | ||
| currentTokens = poolMax | ||
| end | ||
| currentTokens = math.min(currentTokens + owedTokens, poolMax) | ||
| -- Remove the cost and return the new number of tokens | ||
| if currentTokens - cost >= 0 then | ||
| currentTokens = currentTokens - cost | ||
| else | ||
| currentTokens = -1 | ||
| end | ||
| redis.call('hset', key, 'pool', currentTokens) | ||
| redis.call('expire', key, expiry) | ||
| -- Finally return the value - if it's negative then we've hit the limit | ||
| return currentTokens | ||
| ` | ||
|
|
||
| export const createCdpRedisPool = (config: PluginsServerConfig): CdpRedis => { | ||
| const pool = createPool<CdpRedisClient>( | ||
| { | ||
| create: async () => { | ||
| const client = await createRedisClient(config.CDP_REDIS_HOST, { | ||
| port: config.CDP_REDIS_PORT, | ||
| password: config.CDP_REDIS_PASSWORD, | ||
| }) | ||
|
|
||
| client.defineCommand('checkRateLimit', { | ||
| numberOfKeys: 1, | ||
| lua: LUA_TOKEN_BUCKET, | ||
| }) | ||
|
|
||
| return client as CdpRedisClient | ||
| }, | ||
| destroy: async (client) => { | ||
| await client.quit() | ||
| }, | ||
| }, | ||
| { | ||
| min: config.REDIS_POOL_MIN_SIZE, | ||
| max: config.REDIS_POOL_MAX_SIZE, | ||
| autostart: true, | ||
| } | ||
| ) | ||
|
|
||
| const useClient: CdpRedis['useClient'] = async (options, callback) => { | ||
| const timeout = timeoutGuard( | ||
| `Redis call ${options.name} delayed. Waiting over 30 seconds.`, | ||
| undefined, | ||
| options.timeout | ||
| ) | ||
| const client = await pool.acquire() | ||
|
|
||
| try { | ||
| return await callback(client) | ||
| } catch (e) { | ||
| if (options.failOpen) { | ||
| // We log the error and return null | ||
| captureException(e) | ||
| status.error(`Redis call${options.name} failed`, e) | ||
| return null | ||
| } | ||
| throw e | ||
| } finally { | ||
| await pool.release(client) | ||
| clearTimeout(timeout) | ||
| } | ||
| } | ||
|
|
||
| const usePipeline: CdpRedis['usePipeline'] = async (options, callback) => { | ||
| return useClient(options, async (client) => { | ||
| const pipeline = client.pipeline() | ||
| callback(pipeline) | ||
| return pipeline.exec() | ||
| }) | ||
| } | ||
|
|
||
| return { | ||
| useClient, | ||
| usePipeline, | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,12 @@ | ||
| import { CdpRedis } from '../../../src/cdp/redis' | ||
|
|
||
| export async function deleteKeysWithPrefix(redis: CdpRedis, prefix: string) { | ||
| await redis.useClient({ name: 'delete-keys' }, async (client) => { | ||
| const keys = await client.keys(`${prefix}*`) | ||
| const pipeline = client.pipeline() | ||
| keys.forEach(function (key) { | ||
| pipeline.del(key) | ||
| }) | ||
| await pipeline.exec() | ||
| }) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,208 @@ | ||
| jest.mock('../../src/utils/now', () => { | ||
| return { | ||
| now: jest.fn(() => Date.now()), | ||
| } | ||
| }) | ||
| import { BASE_REDIS_KEY, HogMasker } from '../../src/cdp/hog-masker' | ||
| import { CdpRedis, createCdpRedisPool } from '../../src/cdp/redis' | ||
| import { HogFunctionType } from '../../src/cdp/types' | ||
| import { Hub } from '../../src/types' | ||
| import { createHub } from '../../src/utils/db/hub' | ||
| import { delay } from '../../src/utils/utils' | ||
| import { HOG_MASK_EXAMPLES } from './examples' | ||
| import { createHogExecutionGlobals, createHogFunction } from './fixtures' | ||
| import { deleteKeysWithPrefix } from './helpers/redis' | ||
|
|
||
| const mockNow: jest.Mock = require('../../src/utils/now').now as any | ||
|
|
||
| describe('HogMasker', () => { | ||
| describe('integration', () => { | ||
| let now: number | ||
| let hub: Hub | ||
| let closeHub: () => Promise<void> | ||
| let masker: HogMasker | ||
| let redis: CdpRedis | ||
|
|
||
| beforeEach(async () => { | ||
| ;[hub, closeHub] = await createHub() | ||
|
|
||
| now = 1720000000000 | ||
| mockNow.mockReturnValue(now) | ||
|
|
||
| redis = createCdpRedisPool(hub) | ||
| await deleteKeysWithPrefix(redis, BASE_REDIS_KEY) | ||
|
|
||
| masker = new HogMasker(redis) | ||
| }) | ||
|
|
||
| const advanceTime = (ms: number) => { | ||
| now += ms | ||
| mockNow.mockReturnValue(now) | ||
| } | ||
|
|
||
| const reallyAdvanceTime = async (ms: number) => { | ||
| advanceTime(ms) | ||
| await delay(ms) | ||
| } | ||
|
|
||
| afterEach(async () => { | ||
| await closeHub() | ||
| jest.clearAllMocks() | ||
| }) | ||
|
|
||
| it('should return all functions without masks', async () => { | ||
| const normalFunction = createHogFunction({}) | ||
| const invocations = [{ globals: createHogExecutionGlobals(), hogFunction: normalFunction }] | ||
| const res = await masker.filterByMasking(invocations) | ||
|
|
||
| expect(res.notMasked).toHaveLength(1) | ||
| expect(res.masked).toEqual([]) | ||
| }) | ||
|
|
||
| it('should only allow one invocation call when masked for one function', async () => { | ||
| const functionWithAllMasking = createHogFunction({ | ||
| ...HOG_MASK_EXAMPLES.all, | ||
| }) | ||
| const globals1 = createHogExecutionGlobals({ event: { uuid: '1' } as any }) | ||
| const globals2 = createHogExecutionGlobals({ event: { uuid: '2' } as any }) | ||
| const globals3 = createHogExecutionGlobals({ event: { uuid: '3' } as any }) | ||
| const invocations = [ | ||
| { globals: globals1, hogFunction: functionWithAllMasking }, | ||
| { globals: globals2, hogFunction: functionWithAllMasking }, | ||
| { globals: globals3, hogFunction: functionWithAllMasking }, | ||
| ] | ||
|
|
||
| const res = await masker.filterByMasking(invocations) | ||
| expect(res.notMasked).toHaveLength(1) | ||
| expect(res.masked).toHaveLength(2) | ||
| expect(res.notMasked[0].globals).toEqual(globals1) | ||
| expect(res.masked[0].globals).toEqual(globals2) | ||
| expect(res.masked[1].globals).toEqual(globals3) | ||
|
|
||
| const res2 = await masker.filterByMasking(invocations) | ||
| expect(res2.notMasked).toHaveLength(0) | ||
| expect(res2.masked).toHaveLength(3) | ||
| }) | ||
|
|
||
| it('allow multiple functions for the same globals', async () => { | ||
| const functionWithAllMasking = createHogFunction({ | ||
| ...HOG_MASK_EXAMPLES.all, | ||
| }) | ||
| const functionWithAllMasking2 = createHogFunction({ | ||
| ...HOG_MASK_EXAMPLES.all, | ||
| }) | ||
| const functionWithNoMasking = createHogFunction({}) | ||
| const globals = createHogExecutionGlobals() | ||
| const invocations = [ | ||
| { globals, hogFunction: functionWithAllMasking }, | ||
| { globals, hogFunction: functionWithAllMasking2 }, | ||
| { globals, hogFunction: functionWithNoMasking }, | ||
| ] | ||
|
|
||
| const res = await masker.filterByMasking(invocations) | ||
| expect(res.notMasked).toHaveLength(3) | ||
| expect(res.masked).toHaveLength(0) | ||
|
|
||
| const res2 = await masker.filterByMasking(invocations) | ||
| expect(res2.notMasked).toHaveLength(1) | ||
| expect(res2.masked).toHaveLength(2) | ||
| expect(res2.notMasked[0].hogFunction).toEqual(functionWithNoMasking) | ||
| expect(res2.masked[0].hogFunction).toEqual(functionWithAllMasking) | ||
| expect(res2.masked[1].hogFunction).toEqual(functionWithAllMasking2) | ||
| }) | ||
|
|
||
| describe('ttl', () => { | ||
| let hogFunctionPerson: HogFunctionType | ||
| let hogFunctionAll: HogFunctionType | ||
|
|
||
| beforeEach(() => { | ||
| hogFunctionPerson = createHogFunction({ | ||
| masking: { | ||
| ...HOG_MASK_EXAMPLES.person.masking!, | ||
| ttl: 1, | ||
| }, | ||
| }) | ||
|
|
||
| hogFunctionAll = createHogFunction({ | ||
| masking: { | ||
| ...HOG_MASK_EXAMPLES.all.masking!, | ||
| ttl: 1, | ||
| }, | ||
| }) | ||
| }) | ||
| it('should re-allow after the ttl expires', async () => { | ||
| const invocations = [{ globals: createHogExecutionGlobals(), hogFunction: hogFunctionAll }] | ||
| expect((await masker.filterByMasking(invocations)).notMasked).toHaveLength(1) | ||
| expect((await masker.filterByMasking(invocations)).notMasked).toHaveLength(0) | ||
| expect((await masker.filterByMasking(invocations)).notMasked).toHaveLength(0) | ||
| await reallyAdvanceTime(1000) | ||
| expect((await masker.filterByMasking(invocations)).notMasked).toHaveLength(1) | ||
| expect((await masker.filterByMasking(invocations)).notMasked).toHaveLength(0) | ||
| }) | ||
|
|
||
| it('should mask with custom hog hash', async () => { | ||
| const globalsPerson1 = createHogExecutionGlobals({ person: { uuid: '1' } as any }) | ||
| const globalsPerson2 = createHogExecutionGlobals({ person: { uuid: '2' } as any }) | ||
|
|
||
| const invocations = [ | ||
| { globals: globalsPerson1, hogFunction: hogFunctionPerson }, | ||
| { globals: globalsPerson1, hogFunction: hogFunctionAll }, | ||
| { globals: globalsPerson2, hogFunction: hogFunctionPerson }, | ||
| { globals: globalsPerson2, hogFunction: hogFunctionAll }, | ||
| ] | ||
| const res = await masker.filterByMasking(invocations) | ||
| expect(res.masked.length).toEqual(1) | ||
| expect(res.notMasked.length).toEqual(3) | ||
| const res2 = await masker.filterByMasking(invocations) | ||
| expect(res2.masked.length).toEqual(4) | ||
| expect(res2.notMasked.length).toEqual(0) | ||
| }) | ||
|
|
||
| it('should mask until threshold passed', async () => { | ||
| hogFunctionAll.masking!.threshold = 5 | ||
|
|
||
| const invocations = [{ globals: createHogExecutionGlobals(), hogFunction: hogFunctionAll }] | ||
| // First one goes through | ||
| expect((await masker.filterByMasking(invocations)).notMasked).toHaveLength(1) | ||
|
|
||
| // Next 4 should be masked | ||
| expect((await masker.filterByMasking(invocations)).notMasked).toHaveLength(0) | ||
| expect((await masker.filterByMasking(invocations)).notMasked).toHaveLength(0) | ||
| expect((await masker.filterByMasking(invocations)).notMasked).toHaveLength(0) | ||
| expect((await masker.filterByMasking(invocations)).notMasked).toHaveLength(0) | ||
| // Now we have hit the threshold so it should not be masked | ||
| expect((await masker.filterByMasking(invocations)).notMasked).toHaveLength(1) | ||
| // Next 4 should be masked | ||
| expect((await masker.filterByMasking(invocations)).notMasked).toHaveLength(0) | ||
| expect((await masker.filterByMasking(invocations)).notMasked).toHaveLength(0) | ||
| expect((await masker.filterByMasking(invocations)).notMasked).toHaveLength(0) | ||
| expect((await masker.filterByMasking(invocations)).notMasked).toHaveLength(0) | ||
| // Again the Nth one shouldn't be masked | ||
| expect((await masker.filterByMasking(invocations)).notMasked).toHaveLength(1) | ||
| }) | ||
|
|
||
| it('should mask threshold based in a batch', async () => { | ||
| hogFunctionAll.masking!.threshold = 5 | ||
| hogFunctionAll.masking!.ttl = 10 | ||
|
|
||
| // If we have 10 invocations in a batch then we should have 2 invocations that are not masked | ||
| expect( | ||
| ( | ||
| await masker.filterByMasking( | ||
| Array(10).fill({ globals: createHogExecutionGlobals(), hogFunction: hogFunctionAll }) | ||
| ) | ||
| ).notMasked | ||
| ).toHaveLength(2) | ||
|
|
||
| // Next one should cross the threshold | ||
| expect( | ||
| ( | ||
| await masker.filterByMasking([ | ||
| { globals: createHogExecutionGlobals(), hogFunction: hogFunctionAll }, | ||
| ]) | ||
| ).notMasked | ||
| ).toHaveLength(1) | ||
| }) | ||
| }) | ||
| }) | ||
| }) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,17 @@ | ||
| # Generated by Django 4.2.14 on 2024-08-09 09:44 | ||
|
|
||
| from django.db import migrations, models | ||
|
|
||
|
|
||
| class Migration(migrations.Migration): | ||
| dependencies = [ | ||
| ("posthog", "0455_alter_externaldatasource_source_type"), | ||
| ] | ||
|
|
||
| operations = [ | ||
| migrations.AddField( | ||
| model_name="hogfunction", | ||
| name="masking", | ||
| field=models.JSONField(blank=True, null=True), | ||
| ), | ||
| ] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters