diff --git a/docs/source/conf.py b/docs/source/conf.py
index 31ce269ae..949ac5d82 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -87,7 +87,7 @@ autodoc_mock_imports = [
 # Some of our modules cannot generate summary
 autosummary_mock_imports = [
     'nni.retiarii.codegen.tensorflow',
-    'nni.nas.benchmarks.nasbench101.db_gen',
+    'nni.nas.benchmark.nasbench101.db_gen',
     'nni.tools.jupyter_extension.management',
 ] + autodoc_mock_imports
 
diff --git a/docs/source/nas/benchmarks.rst b/docs/source/nas/benchmarks.rst
index 7c0ceddf9..016ccbfb6 100644
--- a/docs/source/nas/benchmarks.rst
+++ b/docs/source/nas/benchmarks.rst
@@ -17,7 +17,7 @@ Data Preparation
 Option 1 (Recommended)
 ^^^^^^^^^^^^^^^^^^^^^^
 
-You can download the preprocessed benchmark files via ``python -m nni.nas.benchmarks.download <benchmark_name>``, where ``<benchmark_name>`` can be ``nasbench101``, ``nasbench201``, and etc. Add ``--help`` to the command for supported command line arguments.
+You can download the preprocessed benchmark files via ``python -m nni.nas.benchmark.download <benchmark_name>``, where ``<benchmark_name>`` can be ``nasbench101``, ``nasbench201``, and etc. Add ``--help`` to the command for supported command line arguments.
 
 Option 2
 ^^^^^^^^
@@ -79,7 +79,7 @@ Instead of storing results obtained with different configurations in separate fi
 
 Here is a list of available operators used in NDS.
 
-.. automodule:: nni.nas.benchmarks.nds.constants
+.. automodule:: nni.nas.benchmark.nds.constants
    :noindex:
 
 See :doc:`example usages </tutorials/nasbench_as_dataset>` and :ref:`API references <nds-reference>`.
diff --git a/examples/nas/benchmarks/nasbench101.sh b/examples/nas/benchmarks/nasbench101.sh
index 606a77f1b..6c5917690 100755
--- a/examples/nas/benchmarks/nasbench101.sh
+++ b/examples/nas/benchmarks/nasbench101.sh
@@ -15,5 +15,5 @@ fi
 echo "Generating database..."
 rm -f ${NASBENCHMARK_DIR}/nasbench101.db ${NASBENCHMARK_DIR}/nasbench101.db-journal
 mkdir -p ${NASBENCHMARK_DIR}
-python3 -m nni.nas.benchmarks.nasbench101.db_gen nasbench_full.tfrecord
+python3 -m nni.nas.benchmark.nasbench101.db_gen nasbench_full.tfrecord
 rm -f nasbench_full.tfrecord
diff --git a/examples/nas/benchmarks/nasbench201.sh b/examples/nas/benchmarks/nasbench201.sh
index 1c4a3df79..c8c4cb168 100755
--- a/examples/nas/benchmarks/nasbench201.sh
+++ b/examples/nas/benchmarks/nasbench201.sh
@@ -15,5 +15,5 @@ fi
 echo "Generating database..."
 rm -f ${NASBENCHMARK_DIR}/nasbench201.db ${NASBENCHMARK_DIR}/nasbench201.db-journal
 mkdir -p ${NASBENCHMARK_DIR}
-python3 -m nni.nas.benchmarks.nasbench201.db_gen a.pth
+python3 -m nni.nas.benchmark.nasbench201.db_gen a.pth
 rm -f a.pth
diff --git a/examples/nas/benchmarks/nds.sh b/examples/nas/benchmarks/nds.sh
index 781ab1007..a39ffca28 100755
--- a/examples/nas/benchmarks/nds.sh
+++ b/examples/nas/benchmarks/nds.sh
@@ -16,5 +16,5 @@ unzip data.zip
 echo "Generating database..."
 rm -f ${NASBENCHMARK_DIR}/nds.db ${NASBENCHMARK_DIR}/nds.db-journal
 mkdir -p ${NASBENCHMARK_DIR}
-python3 -m nni.nas.benchmarks.nds.db_gen nds_data
+python3 -m nni.nas.benchmark.nds.db_gen nds_data
 rm -rf data.zip nds_data
diff --git a/examples/nas/benchmarks/nlp.sh b/examples/nas/benchmarks/nlp.sh
index 49a6b3140..071c32eaa 100644
--- a/examples/nas/benchmarks/nlp.sh
+++ b/examples/nas/benchmarks/nlp.sh
@@ -35,5 +35,5 @@ cd ..
 echo "Generating database..."
 rm -f ${NASBENCHMARK_DIR}/nlp.db ${NASBENCHMARK_DIR}/nlp.db-journal
 mkdir -p ${NASBENCHMARK_DIR}
-python3 -m nni.nas.benchmarks.nlp.db_gen nlp_data
+python3 -m nni.nas.benchmark.nlp.db_gen nlp_data
 rm -rf nlp_data
diff --git a/nni/nas/benchmark/__init__.py b/nni/nas/benchmark/__init__.py
new file mode 100644
index 000000000..e12867e62
--- /dev/null
+++ b/nni/nas/benchmark/__init__.py
@@ -0,0 +1,12 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+
+import warnings
+try:
+    import peewee
+except ImportError:
+    warnings.warn('peewee is not installed. Please install it to use NAS benchmarks.')
+
+# from .evaluator import *
+# from .space import *
+from .utils import load_benchmark, download_benchmark
diff --git a/nni/nas/benchmarks/constants.py b/nni/nas/benchmark/constants.py
similarity index 100%
rename from nni/nas/benchmarks/constants.py
rename to nni/nas/benchmark/constants.py
diff --git a/nni/nas/benchmarks/download.py b/nni/nas/benchmark/download.py
similarity index 100%
rename from nni/nas/benchmarks/download.py
rename to nni/nas/benchmark/download.py
diff --git a/nni/nas/benchmarks/nasbench101/__init__.py b/nni/nas/benchmark/nasbench101/__init__.py
similarity index 71%
rename from nni/nas/benchmarks/nasbench101/__init__.py
rename to nni/nas/benchmark/nasbench101/__init__.py
index bcf1a043f..5578b3f66 100644
--- a/nni/nas/benchmarks/nasbench101/__init__.py
+++ b/nni/nas/benchmark/nasbench101/__init__.py
@@ -2,5 +2,5 @@
 # Licensed under the MIT license.
 
 from .constants import INPUT, OUTPUT, CONV3X3_BN_RELU, CONV1X1_BN_RELU, MAXPOOL3X3
-from .model import Nb101TrialStats, Nb101IntermediateStats, Nb101TrialConfig
+from .schema import Nb101TrialStats, Nb101IntermediateStats, Nb101TrialConfig
 from .query import query_nb101_trial_stats
diff --git a/nni/nas/benchmarks/nasbench101/constants.py b/nni/nas/benchmark/nasbench101/constants.py
similarity index 100%
rename from nni/nas/benchmarks/nasbench101/constants.py
rename to nni/nas/benchmark/nasbench101/constants.py
diff --git a/nni/nas/benchmarks/nasbench101/db_gen.py b/nni/nas/benchmark/nasbench101/db_gen.py
similarity index 95%
rename from nni/nas/benchmarks/nasbench101/db_gen.py
rename to nni/nas/benchmark/nasbench101/db_gen.py
index 79ceb5a99..340e2bb2d 100644
--- a/nni/nas/benchmarks/nasbench101/db_gen.py
+++ b/nni/nas/benchmark/nasbench101/db_gen.py
@@ -6,8 +6,8 @@ import argparse
 from tqdm import tqdm
 from nasbench import api  # pylint: disable=import-error
 
-from nni.nas.benchmarks.utils import load_benchmark
-from .model import Nb101TrialConfig, Nb101TrialStats, Nb101IntermediateStats
+from nni.nas.benchmark.utils import load_benchmark
+from .schema import Nb101TrialConfig, Nb101TrialStats, Nb101IntermediateStats
 from .graph_util import nasbench_format_to_architecture_repr, hash_module
 
 
diff --git a/nni/nas/benchmarks/nasbench101/graph_util.py b/nni/nas/benchmark/nasbench101/graph_util.py
similarity index 100%
rename from nni/nas/benchmarks/nasbench101/graph_util.py
rename to nni/nas/benchmark/nasbench101/graph_util.py
diff --git a/nni/nas/benchmarks/nasbench101/query.py b/nni/nas/benchmark/nasbench101/query.py
similarity index 96%
rename from nni/nas/benchmarks/nasbench101/query.py
rename to nni/nas/benchmark/nasbench101/query.py
index 4f11f78ba..577b6dccc 100644
--- a/nni/nas/benchmarks/nasbench101/query.py
+++ b/nni/nas/benchmark/nasbench101/query.py
@@ -6,8 +6,8 @@ import functools
 from peewee import fn
 from playhouse.shortcuts import model_to_dict
 
-from nni.nas.benchmarks.utils import load_benchmark
-from .model import Nb101TrialStats, Nb101TrialConfig, proxy
+from nni.nas.benchmark.utils import load_benchmark
+from .schema import Nb101TrialStats, Nb101TrialConfig, proxy
 from .graph_util import hash_module, infer_num_vertices
 
 
diff --git a/nni/nas/benchmarks/nasbench101/model.py b/nni/nas/benchmark/nasbench101/schema.py
similarity index 98%
rename from nni/nas/benchmarks/nasbench101/model.py
rename to nni/nas/benchmark/nasbench101/schema.py
index 6e055e3e7..be2ef2906 100644
--- a/nni/nas/benchmarks/nasbench101/model.py
+++ b/nni/nas/benchmark/nasbench101/schema.py
@@ -4,7 +4,7 @@
 from peewee import CharField, FloatField, ForeignKeyField, IntegerField, Model, Proxy
 from playhouse.sqlite_ext import JSONField
 
-from nni.nas.benchmarks.utils import json_dumps
+from nni.nas.benchmark.utils import json_dumps
 
 proxy = Proxy()
 
diff --git a/nni/nas/benchmarks/nasbench201/__init__.py b/nni/nas/benchmark/nasbench201/__init__.py
similarity index 71%
rename from nni/nas/benchmarks/nasbench201/__init__.py
rename to nni/nas/benchmark/nasbench201/__init__.py
index 46e398e77..b7b1ee9e7 100644
--- a/nni/nas/benchmarks/nasbench201/__init__.py
+++ b/nni/nas/benchmark/nasbench201/__init__.py
@@ -2,5 +2,5 @@
 # Licensed under the MIT license.
 
 from .constants import NONE, SKIP_CONNECT, CONV_1X1, CONV_3X3, AVG_POOL_3X3
-from .model import Nb201TrialStats, Nb201IntermediateStats, Nb201TrialConfig
+from .schema import Nb201TrialStats, Nb201IntermediateStats, Nb201TrialConfig
 from .query import query_nb201_trial_stats
diff --git a/nni/nas/benchmarks/nasbench201/constants.py b/nni/nas/benchmark/nasbench201/constants.py
similarity index 100%
rename from nni/nas/benchmarks/nasbench201/constants.py
rename to nni/nas/benchmark/nasbench201/constants.py
diff --git a/nni/nas/benchmarks/nasbench201/db_gen.py b/nni/nas/benchmark/nasbench201/db_gen.py
similarity index 97%
rename from nni/nas/benchmarks/nasbench201/db_gen.py
rename to nni/nas/benchmark/nasbench201/db_gen.py
index 56312734c..8c18ef732 100644
--- a/nni/nas/benchmarks/nasbench201/db_gen.py
+++ b/nni/nas/benchmark/nasbench201/db_gen.py
@@ -7,9 +7,9 @@ import re
 import tqdm
 import torch
 
-from nni.nas.benchmarks.utils import load_benchmark
+from nni.nas.benchmark.utils import load_benchmark
 from .constants import NONE, SKIP_CONNECT, CONV_1X1, CONV_3X3, AVG_POOL_3X3
-from .model import Nb201TrialConfig, Nb201TrialStats, Nb201IntermediateStats
+from .schema import Nb201TrialConfig, Nb201TrialStats, Nb201IntermediateStats
 
 
 def parse_arch_str(arch_str):
diff --git a/nni/nas/benchmarks/nasbench201/query.py b/nni/nas/benchmark/nasbench201/query.py
similarity index 96%
rename from nni/nas/benchmarks/nasbench201/query.py
rename to nni/nas/benchmark/nasbench201/query.py
index 494b293d4..c71b21605 100644
--- a/nni/nas/benchmarks/nasbench201/query.py
+++ b/nni/nas/benchmark/nasbench201/query.py
@@ -6,8 +6,8 @@ import functools
 from peewee import fn
 from playhouse.shortcuts import model_to_dict
 
-from nni.nas.benchmarks.utils import load_benchmark
-from .model import Nb201TrialStats, Nb201TrialConfig, proxy
+from nni.nas.benchmark.utils import load_benchmark
+from .schema import Nb201TrialStats, Nb201TrialConfig, proxy
 
 
 def query_nb201_trial_stats(arch, num_epochs, dataset, reduction=None, include_intermediates=False):
diff --git a/nni/nas/benchmarks/nasbench201/model.py b/nni/nas/benchmark/nasbench201/schema.py
similarity index 99%
rename from nni/nas/benchmarks/nasbench201/model.py
rename to nni/nas/benchmark/nasbench201/schema.py
index 72ed35415..251f26109 100644
--- a/nni/nas/benchmarks/nasbench201/model.py
+++ b/nni/nas/benchmark/nasbench201/schema.py
@@ -4,7 +4,7 @@
 from peewee import CharField, FloatField, ForeignKeyField, IntegerField, Model, Proxy
 from playhouse.sqlite_ext import JSONField
 
-from nni.nas.benchmarks.utils import json_dumps
+from nni.nas.benchmark.utils import json_dumps
 
 proxy = Proxy()
 
diff --git a/nni/nas/benchmarks/nds/__init__.py b/nni/nas/benchmark/nds/__init__.py
similarity index 66%
rename from nni/nas/benchmarks/nds/__init__.py
rename to nni/nas/benchmark/nds/__init__.py
index 51372285c..1c6fd80fe 100644
--- a/nni/nas/benchmarks/nds/__init__.py
+++ b/nni/nas/benchmark/nds/__init__.py
@@ -2,5 +2,5 @@
 # Licensed under the MIT license.
 
 from .constants import *
-from .model import NdsTrialConfig, NdsTrialStats, NdsIntermediateStats
+from .schema import NdsTrialConfig, NdsTrialStats, NdsIntermediateStats
 from .query import query_nds_trial_stats
diff --git a/nni/nas/benchmarks/nds/constants.py b/nni/nas/benchmark/nds/constants.py
similarity index 100%
rename from nni/nas/benchmarks/nds/constants.py
rename to nni/nas/benchmark/nds/constants.py
diff --git a/nni/nas/benchmarks/nds/db_gen.py b/nni/nas/benchmark/nds/db_gen.py
similarity index 97%
rename from nni/nas/benchmarks/nds/db_gen.py
rename to nni/nas/benchmark/nds/db_gen.py
index 26a6c9d69..89d3e72d9 100644
--- a/nni/nas/benchmarks/nds/db_gen.py
+++ b/nni/nas/benchmark/nds/db_gen.py
@@ -8,8 +8,8 @@ import os
 import numpy as np
 import tqdm
 
-from nni.nas.benchmarks.utils import load_benchmark
-from .model import NdsTrialConfig, NdsTrialStats, NdsIntermediateStats
+from nni.nas.benchmark.utils import load_benchmark
+from .schema import NdsTrialConfig, NdsTrialStats, NdsIntermediateStats
 
 
 def inject_item(db, item, proposer, dataset, generator):
diff --git a/nni/nas/benchmarks/nds/query.py b/nni/nas/benchmark/nds/query.py
similarity index 96%
rename from nni/nas/benchmarks/nds/query.py
rename to nni/nas/benchmark/nds/query.py
index cd797c00b..63c40b68e 100644
--- a/nni/nas/benchmarks/nds/query.py
+++ b/nni/nas/benchmark/nds/query.py
@@ -6,8 +6,8 @@ import functools
 from peewee import fn
 from playhouse.shortcuts import model_to_dict
 
-from nni.nas.benchmarks.utils import load_benchmark
-from .model import NdsTrialStats, NdsTrialConfig, proxy
+from nni.nas.benchmark.utils import load_benchmark
+from .schema import NdsTrialStats, NdsTrialConfig, proxy
 
 
 def query_nds_trial_stats(model_family, proposer, generator, model_spec, cell_spec, dataset,
diff --git a/nni/nas/benchmarks/nds/model.py b/nni/nas/benchmark/nds/schema.py
similarity index 99%
rename from nni/nas/benchmarks/nds/model.py
rename to nni/nas/benchmark/nds/schema.py
index 3ceb3a79d..e51a501b7 100644
--- a/nni/nas/benchmarks/nds/model.py
+++ b/nni/nas/benchmark/nds/schema.py
@@ -4,7 +4,7 @@
 from peewee import CharField, FloatField, ForeignKeyField, IntegerField, Model, Proxy
 from playhouse.sqlite_ext import JSONField
 
-from nni.nas.benchmarks.utils import json_dumps
+from nni.nas.benchmark.utils import json_dumps
 
 proxy = Proxy()
 
diff --git a/nni/nas/benchmarks/nlp/__init__.py b/nni/nas/benchmark/nlp/__init__.py
similarity index 61%
rename from nni/nas/benchmarks/nlp/__init__.py
rename to nni/nas/benchmark/nlp/__init__.py
index d3a5f6ff2..ae9db5fbb 100644
--- a/nni/nas/benchmarks/nlp/__init__.py
+++ b/nni/nas/benchmark/nlp/__init__.py
@@ -1,5 +1,5 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT license.
 
-from .model import NlpTrialStats, NlpIntermediateStats, NlpTrialConfig
+from .schema import NlpTrialStats, NlpIntermediateStats, NlpTrialConfig
 from .query import query_nlp_trial_stats
diff --git a/nni/nas/benchmarks/nlp/db_gen.py b/nni/nas/benchmark/nlp/db_gen.py
similarity index 96%
rename from nni/nas/benchmarks/nlp/db_gen.py
rename to nni/nas/benchmark/nlp/db_gen.py
index 603db8e3d..6d1faae89 100644
--- a/nni/nas/benchmarks/nlp/db_gen.py
+++ b/nni/nas/benchmark/nlp/db_gen.py
@@ -6,7 +6,7 @@ import os
 import argparse
 import tqdm
 
-from .model import db, NlpTrialConfig, NlpTrialStats, NlpIntermediateStats
+from .schema import db, NlpTrialConfig, NlpTrialStats, NlpIntermediateStats
 
 def main():
     parser = argparse.ArgumentParser()
diff --git a/nni/nas/benchmarks/nlp/query.py b/nni/nas/benchmark/nlp/query.py
similarity index 96%
rename from nni/nas/benchmarks/nlp/query.py
rename to nni/nas/benchmark/nlp/query.py
index dddd9f1bb..72e5760c7 100644
--- a/nni/nas/benchmarks/nlp/query.py
+++ b/nni/nas/benchmark/nlp/query.py
@@ -5,7 +5,7 @@ import functools
 
 from peewee import fn
 from playhouse.shortcuts import model_to_dict
-from .model import NlpTrialStats, NlpTrialConfig
+from .schema import NlpTrialStats, NlpTrialConfig
 
 def query_nlp_trial_stats(arch, dataset, reduction=None, include_intermediates=False):
     """
diff --git a/nni/nas/benchmarks/nlp/model.py b/nni/nas/benchmark/nlp/schema.py
similarity index 96%
rename from nni/nas/benchmarks/nlp/model.py
rename to nni/nas/benchmark/nlp/schema.py
index aaf7c02ac..94bb01608 100644
--- a/nni/nas/benchmarks/nlp/model.py
+++ b/nni/nas/benchmark/nlp/schema.py
@@ -6,8 +6,8 @@ import os
 from peewee import CharField, FloatField, ForeignKeyField, IntegerField, Model
 from playhouse.sqlite_ext import JSONField, SqliteExtDatabase
 
-from nni.nas.benchmarks.utils import json_dumps
-from nni.nas.benchmarks.constants import DATABASE_DIR
+from nni.nas.benchmark.utils import json_dumps
+from nni.nas.benchmark.constants import DATABASE_DIR
 
 db = SqliteExtDatabase(os.path.join(DATABASE_DIR, 'nlp.db'), autoconnect=True)
 
diff --git a/nni/nas/benchmarks/utils.py b/nni/nas/benchmark/utils.py
similarity index 91%
rename from nni/nas/benchmarks/utils.py
rename to nni/nas/benchmark/utils.py
index 8c1a7027a..fd63becec 100644
--- a/nni/nas/benchmarks/utils.py
+++ b/nni/nas/benchmark/utils.py
@@ -35,7 +35,7 @@ def load_benchmark(benchmark: str) -> SqliteExtDatabase:
         load_or_download_file(local_path, url)
     except FileNotFoundError:
         raise FileNotFoundError(
-            f'Please use `nni.nas.benchmarks.download_benchmark("{benchmark}")` to setup the benchmark first before using it.'
+            f'Please use `nni.nas.benchmark.download_benchmark("{benchmark}")` to setup the benchmark first before using it.'
         )
     _loaded_benchmarks[benchmark] = SqliteExtDatabase(local_path, autoconnect=True)
     return _loaded_benchmarks[benchmark]
diff --git a/nni/nas/benchmarks/__init__.py b/nni/nas/benchmarks/__init__.py
deleted file mode 100644
index 3dd001de7..000000000
--- a/nni/nas/benchmarks/__init__.py
+++ /dev/null
@@ -1,4 +0,0 @@
-# Copyright (c) Microsoft Corporation.
-# Licensed under the MIT license.
-
-from .utils import load_benchmark, download_benchmark
diff --git a/test/algo/nas/benchmark/__init__.py b/test/algo/nas/benchmark/__init__.py
new file mode 100644
index 000000000..e69de29bb