Skip to content

Commit

Permalink
Adding meta dataset files
Browse files Browse the repository at this point in the history
  • Loading branch information
manogna-s committed Jun 17, 2021
1 parent 870cb14 commit 750be5e
Show file tree
Hide file tree
Showing 66 changed files with 53,509 additions and 72 deletions.
5 changes: 5 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@

```commandline
cd checkpoints/pretrained_ckpts/
!wget https://storage.googleapis.com/vit_models/imagenet21k/ViT-B_16.npz
```
72 changes: 72 additions & 0 deletions cdfsl_dataset/configs/meta_dataset_126x126.gin
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
import cdfsl_dataset.meta_dataset_processing
import meta_dataset.data.decoder

# Default values for sampling variable shots / ways.
EpisodeDescriptionConfig.min_ways = 5
EpisodeDescriptionConfig.max_ways_upper_bound = 50
EpisodeDescriptionConfig.max_num_query = 10
EpisodeDescriptionConfig.max_support_set_size = 100 #500
EpisodeDescriptionConfig.max_support_size_contrib_per_class = 10 #100
EpisodeDescriptionConfig.min_log_weight = -0.69314718055994529 # np.log(0.5)
EpisodeDescriptionConfig.max_log_weight = 0.69314718055994529 # np.log(2)
EpisodeDescriptionConfig.ignore_dag_ontology = False
EpisodeDescriptionConfig.ignore_bilevel_ontology = False
EpisodeDescriptionConfig.ignore_hierarchy_probability = 0.0
EpisodeDescriptionConfig.simclr_episode_fraction = 0.0

# Other default values for the data pipeline.
DataConfig.image_height = 126
DataConfig.shuffle_buffer_size = 1000
DataConfig.read_buffer_size_bytes = 1048576 # 1 MB (1024**2)
DataConfig.num_prefetch = 400

meta_dataset_processing.ImageDecoder.image_size = 126

# If we decode features then change the lines below to use FeatureDecoder.
process_episode.support_decoder = @support/meta_dataset_processing.ImageDecoder()
support/meta_dataset_processing.ImageDecoder.data_augmentation = @support/meta_dataset_processing.DataAugmentation()
support/meta_dataset_processing.DataAugmentation.enable_jitter = True
support/meta_dataset_processing.DataAugmentation.jitter_amount = 0
support/meta_dataset_processing.DataAugmentation.enable_gaussian_noise = True
support/meta_dataset_processing.DataAugmentation.gaussian_noise_std = 0.0
support/meta_dataset_processing.DataAugmentation.enable_random_flip = False
support/meta_dataset_processing.DataAugmentation.enable_random_brightness = False
support/meta_dataset_processing.DataAugmentation.random_brightness_delta = 0
support/meta_dataset_processing.DataAugmentation.enable_random_contrast = False
support/meta_dataset_processing.DataAugmentation.random_contrast_delta = 0
support/meta_dataset_processing.DataAugmentation.enable_random_hue = False
support/meta_dataset_processing.DataAugmentation.random_hue_delta = 0
support/meta_dataset_processing.DataAugmentation.enable_random_saturation = False
support/meta_dataset_processing.DataAugmentation.random_saturation_delta = 0

process_episode.query_decoder = @query/meta_dataset_processing.ImageDecoder()
query/meta_dataset_processing.ImageDecoder.data_augmentation = @query/meta_dataset_processing.DataAugmentation()
query/meta_dataset_processing.DataAugmentation.enable_jitter = False
query/meta_dataset_processing.DataAugmentation.jitter_amount = 0
query/meta_dataset_processing.DataAugmentation.enable_gaussian_noise = False
query/meta_dataset_processing.DataAugmentation.gaussian_noise_std = 0.0
query/meta_dataset_processing.DataAugmentation.enable_random_flip = False
query/meta_dataset_processing.DataAugmentation.enable_random_brightness = False
query/meta_dataset_processing.DataAugmentation.random_brightness_delta = 0
query/meta_dataset_processing.DataAugmentation.enable_random_contrast = False
query/meta_dataset_processing.DataAugmentation.random_contrast_delta = 0
query/meta_dataset_processing.DataAugmentation.enable_random_hue = False
query/meta_dataset_processing.DataAugmentation.random_hue_delta = 0
query/meta_dataset_processing.DataAugmentation.enable_random_saturation = False
query/meta_dataset_processing.DataAugmentation.random_saturation_delta = 0

process_batch.batch_decoder = @batch/meta_dataset_processing.ImageDecoder()
batch/meta_dataset_processing.ImageDecoder.data_augmentation = @batch/meta_dataset_processing.DataAugmentation()
batch/meta_dataset_processing.DataAugmentation.enable_jitter = True
batch/meta_dataset_processing.DataAugmentation.jitter_amount = 8
batch/meta_dataset_processing.DataAugmentation.enable_gaussian_noise = True
batch/meta_dataset_processing.DataAugmentation.gaussian_noise_std = 0.0
batch/meta_dataset_processing.DataAugmentation.enable_random_flip = False
batch/meta_dataset_processing.DataAugmentation.enable_random_brightness = True
batch/meta_dataset_processing.DataAugmentation.random_brightness_delta = 0.125
batch/meta_dataset_processing.DataAugmentation.enable_random_contrast = True
batch/meta_dataset_processing.DataAugmentation.random_contrast_delta = 0.2
batch/meta_dataset_processing.DataAugmentation.enable_random_hue = True
batch/meta_dataset_processing.DataAugmentation.random_hue_delta = 0.03
batch/meta_dataset_processing.DataAugmentation.enable_random_saturation = True
batch/meta_dataset_processing.DataAugmentation.random_saturation_delta = 0.2
72 changes: 72 additions & 0 deletions cdfsl_dataset/configs/meta_dataset_224x224.gin
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
import cdfsl_dataset.meta_dataset_processing
import meta_dataset.data.decoder

# Default values for sampling variable shots / ways.
EpisodeDescriptionConfig.min_ways = 5
EpisodeDescriptionConfig.max_ways_upper_bound = 50
EpisodeDescriptionConfig.max_num_query = 10
EpisodeDescriptionConfig.max_support_set_size = 100 #500
EpisodeDescriptionConfig.max_support_size_contrib_per_class = 10 #100
EpisodeDescriptionConfig.min_log_weight = -0.69314718055994529 # np.log(0.5)
EpisodeDescriptionConfig.max_log_weight = 0.69314718055994529 # np.log(2)
EpisodeDescriptionConfig.ignore_dag_ontology = False
EpisodeDescriptionConfig.ignore_bilevel_ontology = False
EpisodeDescriptionConfig.ignore_hierarchy_probability = 0.0
EpisodeDescriptionConfig.simclr_episode_fraction = 0.0

# Other default values for the data pipeline.
DataConfig.image_height = 224
DataConfig.shuffle_buffer_size = 1000
DataConfig.read_buffer_size_bytes = 1048576 # 1 MB (1024**2)
DataConfig.num_prefetch = 400

meta_dataset_processing.ImageDecoder.image_size = 224

# If we decode features then change the lines below to use FeatureDecoder.
process_episode.support_decoder = @support/meta_dataset_processing.ImageDecoder()
support/meta_dataset_processing.ImageDecoder.data_augmentation = @support/meta_dataset_processing.DataAugmentation()
support/meta_dataset_processing.DataAugmentation.enable_jitter = True
support/meta_dataset_processing.DataAugmentation.jitter_amount = 0
support/meta_dataset_processing.DataAugmentation.enable_gaussian_noise = True
support/meta_dataset_processing.DataAugmentation.gaussian_noise_std = 0.0
support/meta_dataset_processing.DataAugmentation.enable_random_flip = False
support/meta_dataset_processing.DataAugmentation.enable_random_brightness = False
support/meta_dataset_processing.DataAugmentation.random_brightness_delta = 0
support/meta_dataset_processing.DataAugmentation.enable_random_contrast = False
support/meta_dataset_processing.DataAugmentation.random_contrast_delta = 0
support/meta_dataset_processing.DataAugmentation.enable_random_hue = False
support/meta_dataset_processing.DataAugmentation.random_hue_delta = 0
support/meta_dataset_processing.DataAugmentation.enable_random_saturation = False
support/meta_dataset_processing.DataAugmentation.random_saturation_delta = 0

process_episode.query_decoder = @query/meta_dataset_processing.ImageDecoder()
query/meta_dataset_processing.ImageDecoder.data_augmentation = @query/meta_dataset_processing.DataAugmentation()
query/meta_dataset_processing.DataAugmentation.enable_jitter = False
query/meta_dataset_processing.DataAugmentation.jitter_amount = 0
query/meta_dataset_processing.DataAugmentation.enable_gaussian_noise = False
query/meta_dataset_processing.DataAugmentation.gaussian_noise_std = 0.0
query/meta_dataset_processing.DataAugmentation.enable_random_flip = False
query/meta_dataset_processing.DataAugmentation.enable_random_brightness = False
query/meta_dataset_processing.DataAugmentation.random_brightness_delta = 0
query/meta_dataset_processing.DataAugmentation.enable_random_contrast = False
query/meta_dataset_processing.DataAugmentation.random_contrast_delta = 0
query/meta_dataset_processing.DataAugmentation.enable_random_hue = False
query/meta_dataset_processing.DataAugmentation.random_hue_delta = 0
query/meta_dataset_processing.DataAugmentation.enable_random_saturation = False
query/meta_dataset_processing.DataAugmentation.random_saturation_delta = 0

process_batch.batch_decoder = @batch/meta_dataset_processing.ImageDecoder()
batch/meta_dataset_processing.ImageDecoder.data_augmentation = @batch/meta_dataset_processing.DataAugmentation()
batch/meta_dataset_processing.DataAugmentation.enable_jitter = True
batch/meta_dataset_processing.DataAugmentation.jitter_amount = 8
batch/meta_dataset_processing.DataAugmentation.enable_gaussian_noise = True
batch/meta_dataset_processing.DataAugmentation.gaussian_noise_std = 0.0
batch/meta_dataset_processing.DataAugmentation.enable_random_flip = False
batch/meta_dataset_processing.DataAugmentation.enable_random_brightness = True
batch/meta_dataset_processing.DataAugmentation.random_brightness_delta = 0.125
batch/meta_dataset_processing.DataAugmentation.enable_random_contrast = True
batch/meta_dataset_processing.DataAugmentation.random_contrast_delta = 0.2
batch/meta_dataset_processing.DataAugmentation.enable_random_hue = True
batch/meta_dataset_processing.DataAugmentation.random_hue_delta = 0.03
batch/meta_dataset_processing.DataAugmentation.enable_random_saturation = True
batch/meta_dataset_processing.DataAugmentation.random_saturation_delta = 0.2
72 changes: 72 additions & 0 deletions cdfsl_dataset/configs/meta_dataset_84x84.gin
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
import cdfsl_dataset.meta_dataset_processing
import meta_dataset.data.decoder

# Default values for sampling variable shots / ways.
EpisodeDescriptionConfig.min_ways = 5
EpisodeDescriptionConfig.max_ways_upper_bound = 50
EpisodeDescriptionConfig.max_num_query = 10
EpisodeDescriptionConfig.max_support_set_size = 100 #500
EpisodeDescriptionConfig.max_support_size_contrib_per_class = 10 #100
EpisodeDescriptionConfig.min_log_weight = -0.69314718055994529 # np.log(0.5)
EpisodeDescriptionConfig.max_log_weight = 0.69314718055994529 # np.log(2)
EpisodeDescriptionConfig.ignore_dag_ontology = False
EpisodeDescriptionConfig.ignore_bilevel_ontology = False
EpisodeDescriptionConfig.ignore_hierarchy_probability = 0.0
EpisodeDescriptionConfig.simclr_episode_fraction = 0.0

# Other default values for the data pipeline.
DataConfig.image_height = 84
DataConfig.shuffle_buffer_size = 1000
DataConfig.read_buffer_size_bytes = 1048576 # 1 MB (1024**2)
DataConfig.num_prefetch = 400

meta_dataset_processing.ImageDecoder.image_size = 84

# If we decode features then change the lines below to use FeatureDecoder.
process_episode.support_decoder = @support/meta_dataset_processing.ImageDecoder()
support/meta_dataset_processing.ImageDecoder.data_augmentation = @support/meta_dataset_processing.DataAugmentation()
support/meta_dataset_processing.DataAugmentation.enable_jitter = True
support/meta_dataset_processing.DataAugmentation.jitter_amount = 0
support/meta_dataset_processing.DataAugmentation.enable_gaussian_noise = True
support/meta_dataset_processing.DataAugmentation.gaussian_noise_std = 0.0
support/meta_dataset_processing.DataAugmentation.enable_random_flip = False
support/meta_dataset_processing.DataAugmentation.enable_random_brightness = False
support/meta_dataset_processing.DataAugmentation.random_brightness_delta = 0
support/meta_dataset_processing.DataAugmentation.enable_random_contrast = False
support/meta_dataset_processing.DataAugmentation.random_contrast_delta = 0
support/meta_dataset_processing.DataAugmentation.enable_random_hue = False
support/meta_dataset_processing.DataAugmentation.random_hue_delta = 0
support/meta_dataset_processing.DataAugmentation.enable_random_saturation = False
support/meta_dataset_processing.DataAugmentation.random_saturation_delta = 0

process_episode.query_decoder = @query/meta_dataset_processing.ImageDecoder()
query/meta_dataset_processing.ImageDecoder.data_augmentation = @query/meta_dataset_processing.DataAugmentation()
query/meta_dataset_processing.DataAugmentation.enable_jitter = False
query/meta_dataset_processing.DataAugmentation.jitter_amount = 0
query/meta_dataset_processing.DataAugmentation.enable_gaussian_noise = False
query/meta_dataset_processing.DataAugmentation.gaussian_noise_std = 0.0
query/meta_dataset_processing.DataAugmentation.enable_random_flip = False
query/meta_dataset_processing.DataAugmentation.enable_random_brightness = False
query/meta_dataset_processing.DataAugmentation.random_brightness_delta = 0
query/meta_dataset_processing.DataAugmentation.enable_random_contrast = False
query/meta_dataset_processing.DataAugmentation.random_contrast_delta = 0
query/meta_dataset_processing.DataAugmentation.enable_random_hue = False
query/meta_dataset_processing.DataAugmentation.random_hue_delta = 0
query/meta_dataset_processing.DataAugmentation.enable_random_saturation = False
query/meta_dataset_processing.DataAugmentation.random_saturation_delta = 0

process_batch.batch_decoder = @batch/meta_dataset_processing.ImageDecoder()
batch/meta_dataset_processing.ImageDecoder.data_augmentation = @batch/meta_dataset_processing.DataAugmentation()
batch/meta_dataset_processing.DataAugmentation.enable_jitter = True
batch/meta_dataset_processing.DataAugmentation.jitter_amount = 8
batch/meta_dataset_processing.DataAugmentation.enable_gaussian_noise = True
batch/meta_dataset_processing.DataAugmentation.gaussian_noise_std = 0.0
batch/meta_dataset_processing.DataAugmentation.enable_random_flip = False
batch/meta_dataset_processing.DataAugmentation.enable_random_brightness = True
batch/meta_dataset_processing.DataAugmentation.random_brightness_delta = 0.125
batch/meta_dataset_processing.DataAugmentation.enable_random_contrast = True
batch/meta_dataset_processing.DataAugmentation.random_contrast_delta = 0.2
batch/meta_dataset_processing.DataAugmentation.enable_random_hue = True
batch/meta_dataset_processing.DataAugmentation.random_hue_delta = 0.03
batch/meta_dataset_processing.DataAugmentation.enable_random_saturation = True
batch/meta_dataset_processing.DataAugmentation.random_saturation_delta = 0.2
20 changes: 20 additions & 0 deletions cdfsl_dataset/meta_dataset/data/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
# coding=utf-8
# Copyright 2021 The Meta-Dataset Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Lint as: python2, python3
"""Sub-module for reading data and assembling episodes."""
# Whether datasets with example-level splits, or pools, are supported.
# Currently, this is not implemented.
POOL_SUPPORTED = False
Loading

0 comments on commit 750be5e

Please sign in to comment.