diff --git a/.clang-format b/.clang-format new file mode 100644 index 00000000..73304266 --- /dev/null +++ b/.clang-format @@ -0,0 +1,88 @@ +--- +AccessModifierOffset: -1 +AlignAfterOpenBracket: AlwaysBreak +AlignConsecutiveAssignments: false +AlignConsecutiveDeclarations: false +AlignEscapedNewlinesLeft: true +AlignOperands: false +AlignTrailingComments: false +AllowAllParametersOfDeclarationOnNextLine: false +AllowShortBlocksOnASingleLine: false +AllowShortCaseLabelsOnASingleLine: false +AllowShortFunctionsOnASingleLine: Empty +AllowShortIfStatementsOnASingleLine: false +AllowShortLoopsOnASingleLine: false +AlwaysBreakAfterReturnType: None +AlwaysBreakBeforeMultilineStrings: true +AlwaysBreakTemplateDeclarations: true +BinPackArguments: false +BinPackParameters: false +BraceWrapping: + AfterClass: false + AfterControlStatement: false + AfterEnum: false + AfterFunction: false + AfterNamespace: false + AfterObjCDeclaration: false + AfterStruct: false + AfterUnion: false + BeforeCatch: false + BeforeElse: false + IndentBraces: false +BreakBeforeBinaryOperators: None +BreakBeforeBraces: Attach +BreakBeforeTernaryOperators: true +BreakConstructorInitializersBeforeComma: false +BreakAfterJavaFieldAnnotations: false +BreakStringLiterals: false +ColumnLimit: 80 +CommentPragmas: '^ IWYU pragma:' +CompactNamespaces: false +ConstructorInitializerAllOnOneLineOrOnePerLine: true +ConstructorInitializerIndentWidth: 4 +ContinuationIndentWidth: 4 +Cpp11BracedListStyle: true +DerivePointerAlignment: false +DisableFormat: false +ForEachMacros: [ FOR_EACH_RANGE, FOR_EACH, ] +IncludeCategories: + - Regex: '^<.*\.h(pp)?>' + Priority: 1 + - Regex: '^<.*' + Priority: 2 + - Regex: '.*' + Priority: 3 +IndentCaseLabels: true +IndentWidth: 2 +IndentWrappedFunctionNames: false +KeepEmptyLinesAtTheStartOfBlocks: false +MacroBlockBegin: '' +MacroBlockEnd: '' +MaxEmptyLinesToKeep: 1 +NamespaceIndentation: None +ObjCBlockIndentWidth: 2 +ObjCSpaceAfterProperty: false +ObjCSpaceBeforeProtocolList: false +PenaltyBreakBeforeFirstCallParameter: 1 +PenaltyBreakComment: 300 +PenaltyBreakFirstLessLess: 120 +PenaltyBreakString: 1000 +PenaltyExcessCharacter: 1000000 +PenaltyReturnTypeOnItsOwnLine: 2000000 +PointerAlignment: Left +ReflowComments: true +SortIncludes: true +SpaceAfterCStyleCast: false +SpaceBeforeAssignmentOperators: true +SpaceBeforeParens: ControlStatements +SpaceInEmptyParentheses: false +SpacesBeforeTrailingComments: 1 +SpacesInAngles: false +SpacesInContainerLiterals: true +SpacesInCStyleCastParentheses: false +SpacesInParentheses: false +SpacesInSquareBrackets: false +Standard: Cpp11 +TabWidth: 8 +UseTab: Never +... diff --git a/.gitignore b/.gitignore index fac58681..2c8070f2 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,5 @@ examples/**/*.pt docs/source/api/ docs/src/ dist/ +.lintbin +.lintbin/** diff --git a/.lintrunner.toml b/.lintrunner.toml index 073ab891..b9fc047a 100644 --- a/.lintrunner.toml +++ b/.lintrunner.toml @@ -3,33 +3,20 @@ code = 'FLAKE8' include_patterns = ['**/*.py'] exclude_patterns = [ '.git/**', - 'build_test_custom_build/**', - 'build/**', - 'caffe2/**', - 'docs/caffe2/**', - 'docs/cpp/src/**', - 'docs/src/**', - 'functorch/docs/**', - 'functorch/examples/**', - 'functorch/notebooks/**', - 'scripts/**', - 'test/generated_type_hints_smoketest.py', - 'third_party/**', - 'torch/include/**', - 'torch/lib/**', - 'venv/**', - '**/*.pyi', - 'tools/test/test_selective_build.py', + '**/build/**', + '**/__pycache__/**', + '**/third_party/**', + 'multipy/runtime/example/generated/**', ] command = [ 'python3', - 'tools/linter/adapters/flake8_linter.py', + 'scripts/linter/adapters/flake8_linter.py', '--', '@{{PATHSFILE}}' ] init_command = [ 'python3', - 'tools/linter/adapters/pip_init.py', + 'scripts/linter/adapters/pip_init.py', '--dry-run={{DRYRUN}}', 'flake8==3.8.2', 'flake8-bugbear==20.1.4', @@ -45,26 +32,20 @@ init_command = [ [[linter]] code = 'CLANGFORMAT' include_patterns = [ - 'aten/src/ATen/*.h', - 'aten/src/ATen/native/vulkan/**/*.h', - 'aten/src/ATen/native/vulkan/**/*.cpp', - 'c10/**/*.h', - 'c10/**/*.cpp', - 'torch/csrc/**/*.h', - 'torch/csrc/**/*.cpp', - 'test/cpp/**/*.h', - 'test/cpp/**/*.cpp', + '**/*.h', + '**/*.cpp', ] exclude_patterns = [ - 'aten/src/ATen/native/vulkan/api/vk_mem_alloc.h', - 'c10/util/strong_type.h', - 'torch/csrc/jit/serialization/mobile_bytecode_generated.h', - 'torch/csrc/utils/pythoncapi_compat.h', + '.git/**', + '**/build/**', + '**/__pycache__/**', + '**/third_party/**', + 'multipy/runtime/example/generated/**', ] init_command = [ 'python3', - 'tools/linter/adapters/s3_init.py', - '--config-json=tools/linter/adapters/s3_init_config.json', + 'scripts/linter/adapters/s3_init.py', + '--config-json=scripts/linter/adapters/s3_init_config.json', '--linter=clang-format', '--dry-run={{DRYRUN}}', '--output-dir=.lintbin', @@ -72,7 +53,7 @@ init_command = [ ] command = [ 'python3', - 'tools/linter/adapters/clangformat_linter.py', + 'scripts/linter/adapters/clangformat_linter.py', '--binary=.lintbin/clang-format', '--', '@{{PATHSFILE}}' @@ -81,62 +62,26 @@ is_formatter = true [[linter]] code = 'MYPY' -include_patterns = [ - 'torch/**/*.py', - 'torch/**/*.pyi', - 'caffe2/**/*.py', - 'caffe2/**/*.pyi', - 'test/test_bundled_images.py', - 'test/test_bundled_inputs.py', - 'test/test_complex.py', - 'test/test_datapipe.py', - 'test/test_futures.py', - # 'test/test_numpy_interop.py', - 'test/test_torch.py', - 'test/test_type_hints.py', - 'test/test_type_info.py', - 'test/test_utils.py', -] +include_patterns = ['**/*.py'] exclude_patterns = [ - 'torch/include/**', - 'torch/csrc/**', - 'torch/_dynamo/**/*.py', - 'torch/_inductor/**/*.py', - 'torch/_functorch/aot_autograd.py', - 'torch/_functorch/benchmark_utils.py', - 'torch/_functorch/compile_utils.py', - 'torch/_functorch/compilers.py', - 'torch/_functorch/eager_transforms.py', - 'torch/_functorch/fx_minifier.py', - 'torch/_functorch/partitioners.py', - 'torch/_functorch/make_functional.py', - 'torch/_functorch/top_operators_github_usage.py', - 'torch/_functorch/vmap.py', - 'torch/distributed/elastic/agent/server/api.py', - 'torch/testing/_internal/**', - 'torch/distributed/fsdp/fully_sharded_data_parallel.py', - 'torch/distributed/distributed_c10d.py', - # TODO(suo): these exclusions were added just to get lint clean on master. - # Follow up to do more target suppressions and remove them. - 'torch/ao/quantization/fx/convert.py', - 'torch/ao/quantization/_dbr/function_fusion.py', - 'test/test_datapipe.py', - 'caffe2/contrib/fakelowp/test/test_batchmatmul_nnpi_fp16.py', - 'test/test_numpy_interop.py', - 'torch/torch_version.py', + '.git/**', + '**/build/**', + '**/__pycache__/**', + '**/third_party/**', + 'multipy/runtime/example/generated/**', ] command = [ 'python3', - 'tools/linter/adapters/mypy_linter.py', + 'scripts/linter/adapters/mypy_linter.py', '--config=mypy.ini', '--', '@{{PATHSFILE}}' ] init_command = [ 'python3', - 'tools/linter/adapters/pip_init.py', + 'scripts/linter/adapters/pip_init.py', '--dry-run={{DRYRUN}}', - 'numpy==1.23.1', + 'numpy==1.21.6', 'expecttest==0.1.3', 'mypy==0.960', 'types-requests==2.27.25', @@ -151,99 +96,23 @@ init_command = [ 'pyyaml==6.0', ] -[[linter]] -code = 'MYPYNOFOLLOW' -include_patterns = [ - 'torch/_dynamo/eval_frame.py', - 'torch/_dynamo/convert_frame.py', - 'torch/_dynamo/symbolic_convert.py', - 'torch/_dynamo/types.py', - 'torch/_dynamo/output_graph.py', - 'torch/_dynamo/guards.py', - 'torch/_dynamo/side_effects.py', - 'torch/_dynamo/optimizations/__init__.py', - 'torch/_dynamo/optimizations/backends.py', - 'torch/_dynamo/optimizations/training.py', - 'torch/_inductor/graph.py', - 'torch/_C/_dynamo/**/*.py', -] -exclude_patterns = [ -] -command = [ - 'python3', - 'tools/linter/adapters/mypy_linter.py', - '--config=mypy-nofollow.ini', - '--code=MYPYNOFOLLOW', - '--', - '@{{PATHSFILE}}' -] - -[[linter]] -code = 'MYPYSTRICT' -include_patterns = [ - '.github/**/*.py', - 'benchmarks/instruction_counts/**/*.py', - 'tools/**/*.py', - 'torchgen/**/*.py', - 'torch/utils/_pytree.py', - 'torch/utils/benchmark/utils/common.py', - 'torch/utils/benchmark/utils/timer.py', - 'torch/utils/benchmark/utils/valgrind_wrapper/**/*.py', -] -exclude_patterns = [ - # (linbinyu) copied from internal repo - 'tools/code_analyzer/gen_operators_yaml.py', - 'tools/dynamo/verify_dynamo.py', - 'tools/gen_vulkan_spv.py', - 'tools/test/gen_operators_yaml_test.py', - 'tools/test/gen_oplist_test.py', - 'tools/test/test_selective_build.py', -] -command = [ - 'python3', - 'tools/linter/adapters/mypy_linter.py', - '--config=mypy-strict.ini', - '--code=MYPYSTRICT', - '--', - '@{{PATHSFILE}}' -] - [[linter]] code = 'CLANGTIDY' include_patterns = [ - 'c10/core/**/*.cpp', - 'torch/csrc/fx/**/*.cpp', - 'torch/csrc/generic/**/*.cpp', - 'torch/csrc/onnx/**/*.cpp', - 'torch/csrc/tensor/**/*.cpp', - 'torch/csrc/utils/**/*.cpp', + 'multipy/runtime/**/*.h', + 'multipy/runtime/**/*.cpp', ] exclude_patterns = [ - # The negative filters below are to exclude files that include onnx_pb.h or - # caffe2_pb.h, otherwise we'd have to build protos as part of this CI job. - # FunctionsManual.cpp is excluded to keep this diff clean. It will be fixed - # in a follow up PR. - # /torch/csrc/generic/*.cpp is excluded because those files aren't actually built. - # that are not easily converted to accepted c++ - 'c10/test/**/*.cpp', - 'torch/csrc/jit/passes/onnx/helper.cpp', - 'torch/csrc/jit/passes/onnx/shape_type_inference.cpp', - 'torch/csrc/jit/serialization/onnx.cpp', - 'torch/csrc/jit/serialization/export.cpp', - 'torch/csrc/jit/serialization/import.cpp', - 'torch/csrc/jit/serialization/import_legacy.cpp', - 'torch/csrc/onnx/init.cpp', - 'torch/csrc/cuda/nccl.*', - 'torch/csrc/cuda/python_nccl.cpp', - 'torch/csrc/autograd/FunctionsManual.cpp', - 'torch/csrc/generic/*.cpp', - 'torch/csrc/jit/codegen/cuda/runtime/*', - 'torch/csrc/utils/disable_torch_function.cpp', + '.git/**', + '**/build/**', + '**/__pycache__/**', + '**/third_party/**', + 'multipy/runtime/example/generated/**', ] init_command = [ 'python3', - 'tools/linter/adapters/s3_init.py', - '--config-json=tools/linter/adapters/s3_init_config.json', + 'scripts/linter/adapters/s3_init.py', + '--config-json=scripts/linter/adapters/s3_init_config.json', '--linter=clang-tidy', '--dry-run={{DRYRUN}}', '--output-dir=.lintbin', @@ -251,22 +120,26 @@ init_command = [ ] command = [ 'python3', - 'tools/linter/adapters/clangtidy_linter.py', + 'scripts/linter/adapters/clangtidy_linter.py', '--binary=.lintbin/clang-tidy', - '--build_dir=./build', + '--build_dir=./multipy/runtime/build', '--', '@{{PATHSFILE}}' ] [[linter]] code = 'TYPEIGNORE' -include_patterns = ['**/*.py', '**/*.pyi'] +include_patterns = ['**/*.py'] exclude_patterns = [ - 'test/test_jit.py', + '.git/**', + '**/build/**', + '**/__pycache__/**', + '**/third_party/**', + 'multipy/runtime/example/generated/**', ] command = [ 'python3', - 'tools/linter/adapters/grep_linter.py', + 'scripts/linter/adapters/grep_linter.py', '--pattern=# type:\s*ignore([^\[]|$)', '--linter-name=TYPEIGNORE', '--error-name=unqualified type: ignore', @@ -278,69 +151,20 @@ command = [ '@{{PATHSFILE}}' ] -[[linter]] -code = 'NOQA' -include_patterns = ['**/*.py', '**/*.pyi'] -exclude_patterns = ['caffe2/**'] -command = [ - 'python3', - 'tools/linter/adapters/grep_linter.py', - '--pattern=# noqa([^:]|$)', - '--linter-name=NOQA', - '--error-name=unqualified noqa', - """--error-description=\ - This line has an unqualified `noqa`; \ - please convert it to `noqa: XXXX`\ - """, - '--', - '@{{PATHSFILE}}' -] - -[[linter]] -code = 'CIRCLECI' -include_patterns=['.circleci/**'] -command = [ - 'python3', - 'tools/linter/adapters/circleci_linter.py', - '--regen-script-working-dir=.circleci', - '--config-yml=.circleci/config.yml', - '--regen-script=generate_config_yml.py', -] - -[[linter]] -code = 'NATIVEFUNCTIONS' -include_patterns=['aten/src/ATen/native/native_functions.yaml'] -command = [ - 'python3', - 'tools/linter/adapters/nativefunctions_linter.py', - '--native-functions-yml=aten/src/ATen/native/native_functions.yaml', -] -init_command = [ - 'python3', - 'tools/linter/adapters/pip_init.py', - '--dry-run={{DRYRUN}}', - 'ruamel.yaml==0.17.4', -] -is_formatter = true [[linter]] code = 'NEWLINE' include_patterns=['**'] exclude_patterns=[ - '**/contrib/**', - 'third_party/**', - '**/*.bat', - '**/*.expect', - '**/*.ipynb', - '**/*.ps1', - '**/*.ptl', - 'tools/clang_format_hash/**', - 'test/cpp/jit/upgrader_models/*.ptl', - 'test/cpp/jit/upgrader_models/*.ptl.ff', + '.git/**', + '**/build/**', + '**/__pycache__/**', + '.lintbin/**', + '.clang-format', ] command = [ 'python3', - 'tools/linter/adapters/newlines_linter.py', + 'scripts/linter/adapters/newlines_linter.py', '--', '@{{PATHSFILE}}', ] @@ -350,16 +174,16 @@ is_formatter = true code = 'SPACES' include_patterns = ['**'] exclude_patterns = [ - '**/contrib/**', - '**/*.diff', - 'third_party/**', - 'aten/src/ATen/native/vulkan/api/vk_mem_alloc.h', - 'test/cpp/jit/upgrader_models/*.ptl', - 'test/cpp/jit/upgrader_models/*.ptl.ff', + '.git/**', + '**/build/**', + '**/__pycache__/**', + '**/third_party/**', + '.clang-format', + 'multipy/runtime/example/generated/**', ] command = [ 'python3', - 'tools/linter/adapters/grep_linter.py', + 'scripts/linter/adapters/grep_linter.py', '--pattern=[[:blank:]]$', '--linter-name=SPACES', '--error-name=trailing spaces', @@ -375,23 +199,21 @@ command = [ code = 'TABS' include_patterns = ['**'] exclude_patterns = [ - '**/*.svg', - '**/*Makefile', - '**/*Makefile_dashboard', - '**/contrib/**', - 'third_party/**', - '**/.gitattributes', - '**/.gitmodules', - 'aten/src/ATen/native/vulkan/api/vk_mem_alloc.h', - 'test/cpp/jit/upgrader_models/*.ptl', - 'test/cpp/jit/upgrader_models/*.ptl.ff', + '.git/**', + '**/build/**', + '**/__pycache__/**', + '**/third_party/**', + 'scripts/linter/adapters/**', + 'multipy/runtime/example/generated/**', '.lintrunner.toml', + '.clang-format', + 'README.md', ] command = [ 'python3', - 'tools/linter/adapters/grep_linter.py', + 'scripts/linter/adapters/grep_linter.py', # @lint-ignore TXT2 - '--pattern= ', + '--pattern= ', '--linter-name=TABS', '--error-name=saw some tabs', '--replace-pattern=s/\t/ /', @@ -405,20 +227,19 @@ command = [ [[linter]] code = 'INCLUDE' include_patterns = [ - 'c10/**', - 'aten/**', - 'torch/csrc/**', + '**/*.cpp', + '**/*.h', ] exclude_patterns = [ - 'aten/src/ATen/native/quantized/cpu/qnnpack/**', - 'aten/src/ATen/native/vulkan/api/vk_mem_alloc.h', - 'aten/src/ATen/native/vulkan/glsl/**', - 'torch/csrc/jit/serialization/mobile_bytecode_generated.h', - 'torch/csrc/utils/pythoncapi_compat.h', + '.git/**', + '**/build/**', + '**/__pycache__/**', + '**/third_party/**', + 'multipy/runtime/example/generated/**', ] command = [ 'python3', - 'tools/linter/adapters/grep_linter.py', + 'scripts/linter/adapters/grep_linter.py', '--pattern=#include "', '--linter-name=INCLUDE', '--error-name=quoted include', @@ -437,13 +258,15 @@ include_patterns = [ '**/*.h', ] exclude_patterns = [ - 'torch/csrc/utils/pybind.h', - 'torch/utils/benchmark/utils/valgrind_wrapper/compat_bindings.cpp', - 'caffe2/**/*', + '.git/**', + '**/build/**', + '**/__pycache__/**', + '**/third_party/**', + 'multipy/runtime/example/generated/**', ] command = [ 'python3', - 'tools/linter/adapters/grep_linter.py', + 'scripts/linter/adapters/grep_linter.py', '--pattern=#include ', '--linter-name=PYBIND11_INCLUDE', @@ -463,35 +286,6 @@ command = [ '@{{PATHSFILE}}' ] -[[linter]] -code = 'ERROR_PRONE_ISINSTANCE' -include_patterns = [ - 'torch/_refs/**/*.py', - 'torch/_prims/**/*.py', - 'torch/_prims_common/**/*.py', - 'torch/_decomp/**/*.py', - 'torch/_meta_registrations.py', -] -command = [ - 'python3', - 'tools/linter/adapters/grep_linter.py', - '--pattern=isinstance\([^)]+(int|float)\)', - '--linter-name=ERROR_PRONE_ISINSTANCE', - '--error-name=error prone isinstance', - """--error-description=\ - This line has an isinstance call that directly refers to \ - int or float. This is error-prone because you may also \ - have wanted to allow SymInt or SymFloat in your test. \ - To suppress this lint, use an appropriate type alias defined \ - in torch._prims_common; use IntLike/FloatLike when you would accept \ - both regular and symbolic numbers, Dim for ints representing \ - dimensions, or IntWithoutSymInt/FloatWithoutSymFloat if you really \ - meant to exclude symbolic numbers. - """, - '--', - '@{{PATHSFILE}}' -] - [[linter]] code = 'PYBIND11_SPECIALIZATION' include_patterns = [ @@ -499,17 +293,15 @@ include_patterns = [ '**/*.h', ] exclude_patterns = [ - # The place for all orphan specializations - 'torch/csrc/utils/pybind.h', - # These specializations are non-orphan - 'torch/csrc/distributed/c10d/init.cpp', - 'torch/csrc/jit/python/pybind.h', - # These are safe to exclude as they do not have Python - 'c10/**/*', + '.git/**', + '**/build/**', + '**/__pycache__/**', + '**/third_party/**', + 'multipy/runtime/example/generated/**', ] command = [ 'python3', - 'tools/linter/adapters/grep_linter.py', + 'scripts/linter/adapters/grep_linter.py', '--pattern=PYBIND11_DECLARE_HOLDER_TYPE', '--linter-name=PYBIND11_SPECIALIZATION', '--error-name=pybind11 specialization in non-standard location', @@ -537,7 +329,7 @@ exclude_patterns = [ ] command = [ 'python3', - 'tools/linter/adapters/grep_linter.py', + 'scripts/linter/adapters/grep_linter.py', """--pattern=\ (pip|pip3|python -m pip|python3 -m pip|python3 -mpip|python -mpip) \ install ([a-zA-Z0-9][A-Za-z0-9\\._\\-]+)([^/=<>~!]+)[A-Za-z0-9\\._\\-\\*\\+\\!]*$\ @@ -556,98 +348,19 @@ command = [ code = 'EXEC' include_patterns = ['**'] exclude_patterns = [ - 'third_party/**', - 'torch/bin/**', - '**/*.so', - '**/*.py', - '**/*.sh', - '**/*.bash', - '**/git-pre-commit', - '**/git-clang-format', - '**/gradlew', + '.git/**', + '**/build/**', + '**/__pycache__/**', + '**/third_party/**', + 'multipy/runtime/example/generated/**', ] command = [ 'python3', - 'tools/linter/adapters/exec_linter.py', + 'scripts/linter/adapters/exec_linter.py', '--', '@{{PATHSFILE}}', ] -[[linter]] -code = 'CUBINCLUDE' -include_patterns = ['aten/**'] -exclude_patterns = [ - 'aten/src/ATen/cuda/cub*.cuh', -] -command = [ - 'python3', - 'tools/linter/adapters/grep_linter.py', - '--pattern=#include str: - return name.replace("\\", "/") if IS_WINDOWS else name - - -def run_command(args: List[str], cwd: str) -> "subprocess.CompletedProcess[bytes]": - logging.debug("$ %s", " ".join(args)) - start_time = time.monotonic() - try: - return subprocess.run( - args, - cwd=cwd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - check=True, - ) - finally: - end_time = time.monotonic() - logging.debug("took %dms", (end_time - start_time) * 1000) - - -def run_check( - regen_script_working_dir: str, regen_script: str, config_file: str -) -> List[LintMessage]: - try: - proc = run_command(["python3", regen_script], regen_script_working_dir) - except Exception as err: - return [ - LintMessage( - path=None, - line=None, - char=None, - code="CIRCLECI", - severity=LintSeverity.ERROR, - name="command-failed", - original=None, - replacement=None, - description=( - f"Failed due to {err.__class__.__name__}:\n{err}" - if not isinstance(err, subprocess.CalledProcessError) - else ( - "COMMAND (exit code {returncode})\n" - "{command}\n\n" - "STDERR\n{stderr}\n\n" - "STDOUT\n{stdout}" - ).format( - returncode=err.returncode, - command=" ".join(as_posix(x) for x in err.cmd), - stderr=err.stderr.decode("utf-8").strip() or "(empty)", - stdout=err.stdout.decode("utf-8").strip() or "(empty)", - ) - ), - ) - ] - - with open(config_file, mode="rb") as f: - config = f.read() - if proc.stdout == config: - return [] - - return [ - LintMessage( - path=config_file, - line=None, - char=None, - code="CIRCLECI", - severity=LintSeverity.ERROR, - name="config inconsistency", - original=config.decode("utf-8"), - replacement=proc.stdout.decode("utf-8"), - description=( - "The checked-in CircleCI config.yml file does not match what was generated by the scripts. " - "Re-run with '-a' to accept changes." - ), - ) - ] - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="circleci consistency linter", - fromfile_prefix_chars="@", - ) - parser.add_argument( - "--config-yml", - required=True, - help="location of config.yml", - ) - parser.add_argument( - "--regen-script-working-dir", - required=True, - help="this script will chdir to this argument before running --regen-script", - ) - parser.add_argument( - "--regen-script", - required=True, - help="location of the config generation script, relative to --regen-script-working-dir", - ) - parser.add_argument( - "--verbose", - action="store_true", - help="verbose logging", - ) - - args = parser.parse_args() - - logging.basicConfig( - format="<%(threadName)s:%(levelname)s> %(message)s", - level=logging.NOTSET if args.verbose else logging.DEBUG, - stream=sys.stderr, - ) - - for lint_message in run_check( - args.regen_script_working_dir, args.regen_script, args.config_yml - ): - print(json.dumps(lint_message._asdict()), flush=True) diff --git a/scripts/linter/adapters/mypy_linter.py b/scripts/linter/adapters/mypy_linter.py index cd94879f..65ee8850 100644 --- a/scripts/linter/adapters/mypy_linter.py +++ b/scripts/linter/adapters/mypy_linter.py @@ -87,7 +87,6 @@ def check_files( filenames: List[str], config: str, retries: int, - code: str, ) -> List[LintMessage]: try: proc = run_command( @@ -101,7 +100,7 @@ def check_files( path=None, line=None, char=None, - code=code, + code="MYPY", severity=LintSeverity.ERROR, name="command-failed", original=None, @@ -119,7 +118,7 @@ def check_files( char=int(match["column"]) if match["column"] is not None and not match["column"].startswith("-") else None, - code=code, + code="MYPY", severity=severities.get(match["severity"], LintSeverity.ERROR), original=None, replacement=None, @@ -144,11 +143,6 @@ def main() -> None: required=True, help="path to an mypy .ini config file", ) - parser.add_argument( - "--code", - default="MYPY", - help="the code this lint should report as", - ) parser.add_argument( "--verbose", action="store_true", @@ -188,7 +182,7 @@ def main() -> None: else: filenames[filename] = True - lint_messages = check_files(list(filenames), args.config, args.retries, args.code) + lint_messages = check_files(list(filenames), args.config, args.retries) for lint_message in lint_messages: print(json.dumps(lint_message._asdict()), flush=True) diff --git a/scripts/linter/adapters/nativefunctions_linter.py b/scripts/linter/adapters/nativefunctions_linter.py deleted file mode 100644 index 12a6c7e0..00000000 --- a/scripts/linter/adapters/nativefunctions_linter.py +++ /dev/null @@ -1,107 +0,0 @@ -#!/usr/bin/env python3 -""" -Verify that it is possible to round-trip native_functions.yaml via ruamel under some -configuration. Keeping native_functions.yaml consistent in this way allows us to -run codemods on the file using ruamel without introducing line noise. Note that we don't -want to normalize the YAML file, as that would to lots of spurious lint failures. Anything -that ruamel understands how to roundtrip, e.g., whitespace and comments, is OK! - -ruamel is a bit picky about inconsistent indentation, so you will have to indent your -file properly. Also, if you are working on changing the syntax of native_functions.yaml, -you may find that you want to use some format that is not what ruamel prefers. If so, -it is OK to modify this script (instead of reformatting native_functions.yaml)--the point -is simply to make sure that there is *some* configuration of ruamel that can round trip -the YAML, not to be prescriptive about it. -""" - -import argparse -import json -import sys -from enum import Enum -from io import StringIO -from typing import NamedTuple, Optional - -import ruamel.yaml # type: ignore[import] - - -class LintSeverity(str, Enum): - ERROR = "error" - WARNING = "warning" - ADVICE = "advice" - DISABLED = "disabled" - - -class LintMessage(NamedTuple): - path: Optional[str] - line: Optional[int] - char: Optional[int] - code: str - severity: LintSeverity - name: str - original: Optional[str] - replacement: Optional[str] - description: Optional[str] - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="native functions linter", - fromfile_prefix_chars="@", - ) - parser.add_argument( - "--native-functions-yml", - required=True, - help="location of native_functions.yaml", - ) - - args = parser.parse_args() - - with open(args.native_functions_yml) as f: - contents = f.read() - - yaml = ruamel.yaml.YAML() # type: ignore[attr-defined] - yaml.preserve_quotes = True # type: ignore[assignment] - yaml.width = 1000 # type: ignore[assignment] - yaml.boolean_representation = ["False", "True"] # type: ignore[attr-defined] - try: - r = yaml.load(contents) - except Exception as err: - msg = LintMessage( - path=None, - line=None, - char=None, - code="NATIVEFUNCTIONS", - severity=LintSeverity.ERROR, - name="YAML load failure", - original=None, - replacement=None, - description=f"Failed due to {err.__class__.__name__}:\n{err}", - ) - - print(json.dumps(msg._asdict()), flush=True) - sys.exit(0) - - # Cuz ruamel's author intentionally didn't include conversion to string - # https://stackoverflow.com/questions/47614862/best-way-to-use-ruamel-yaml-to-dump-to-string-not-to-stream - string_stream = StringIO() - yaml.dump(r, string_stream) - new_contents = string_stream.getvalue() - string_stream.close() - - if contents != new_contents: - msg = LintMessage( - path=args.native_functions_yml, - line=None, - char=None, - code="NATIVEFUNCTIONS", - severity=LintSeverity.ERROR, - name="roundtrip inconsistency", - original=contents, - replacement=new_contents, - description=( - "YAML roundtrip failed; run `lintrunner --take NATIVEFUNCTIONS -a` to apply the suggested changes. " - "If you think this is in error, please see tools/linter/adapters/nativefunctions_linter.py" - ), - ) - - print(json.dumps(msg._asdict()), flush=True) diff --git a/scripts/linter/adapters/testowners_linter.py b/scripts/linter/adapters/testowners_linter.py old mode 100755 new mode 100644 diff --git a/scripts/linter/adapters/update_s3.py b/scripts/linter/adapters/update_s3.py deleted file mode 100644 index 5f19b472..00000000 --- a/scripts/linter/adapters/update_s3.py +++ /dev/null @@ -1,95 +0,0 @@ -"""Uploads a new binary to s3 and updates its hash in the config file. - -You'll need to have appropriate credentials on the PyTorch AWS buckets, see: -https://boto3.amazonaws.com/v1/documentation/api/latest/guide/quickstart.html#configuration -for how to configure them. -""" - -import argparse -import hashlib -import json -import logging -import os - -import boto3 # type: ignore[import] - - -def compute_file_sha256(path: str) -> str: - """Compute the SHA256 hash of a file and return it as a hex string.""" - # If the file doesn't exist, return an empty string. - if not os.path.exists(path): - return "" - - hash = hashlib.sha256() - - # Open the file in binary mode and hash it. - with open(path, "rb") as f: - for b in f: - hash.update(b) - - # Return the hash as a hexadecimal string. - return hash.hexdigest() - - -def main() -> None: - parser = argparse.ArgumentParser( - description="s3 binary updater", - fromfile_prefix_chars="@", - ) - parser.add_argument( - "--config-json", - required=True, - help="path to config json that you are trying to update", - ) - parser.add_argument( - "--linter", - required=True, - help="name of linter you're trying to update", - ) - parser.add_argument( - "--platform", - required=True, - help="which platform you are uploading the binary for", - ) - parser.add_argument( - "--file", - required=True, - help="file to upload", - ) - parser.add_argument( - "--dry-run", - action="store_true", - help="if set, don't actually upload/write hash", - ) - args = parser.parse_args() - logging.basicConfig(level=logging.INFO) - - config = json.load(open(args.config_json)) - linter_config = config[args.linter][args.platform] - bucket = linter_config["s3_bucket"] - object_name = linter_config["object_name"] - - # Upload the file - logging.info( - f"Uploading file {args.file} to s3 bucket: {bucket}, object name: {object_name}" - ) - if not args.dry_run: - s3_client = boto3.client("s3") - s3_client.upload_file(args.file, bucket, object_name) - - # Update hash in repo - hash_of_new_binary = compute_file_sha256(args.file) - logging.info(f"Computed new hash for binary {hash_of_new_binary}") - - linter_config["hash"] = hash_of_new_binary - config_dump = json.dumps(config, indent=4, sort_keys=True) - - logging.info("Writing out new config:") - logging.info(config_dump) - if not args.dry_run: - with open(args.config_json, "w") as f: - f.write(config_dump) - - -if __name__ == "__main__": - main() diff --git a/scripts/linter/adapters/workflow_consistency_linter.py b/scripts/linter/adapters/workflow_consistency_linter.py deleted file mode 100644 index 6e5fb4db..00000000 --- a/scripts/linter/adapters/workflow_consistency_linter.py +++ /dev/null @@ -1,115 +0,0 @@ -"""Checks for consistency of jobs between different GitHub workflows. - -Any job with a specific `sync-tag` must match all other jobs with the same `sync-tag`. -""" -import argparse -import itertools -import json -from collections import defaultdict -from enum import Enum -from pathlib import Path -from typing import Any, Dict, Iterable, NamedTuple, Optional - -from yaml import CSafeLoader, dump, load - - -class LintSeverity(str, Enum): - ERROR = "error" - WARNING = "warning" - ADVICE = "advice" - DISABLED = "disabled" - - -class LintMessage(NamedTuple): - path: Optional[str] - line: Optional[int] - char: Optional[int] - code: str - severity: LintSeverity - name: str - original: Optional[str] - replacement: Optional[str] - description: Optional[str] - - -def glob_yamls(path: Path) -> Iterable[Path]: - return itertools.chain(path.glob("**/*.yml"), path.glob("**/*.yaml")) - - -def load_yaml(path: Path) -> Any: - with open(path) as f: - return load(f, CSafeLoader) - - -def is_workflow(yaml: Any) -> bool: - return yaml.get("jobs") is not None - - -def print_lint_message(path: Path, job: Dict[str, Any], sync_tag: str) -> None: - job_id = list(job.keys())[0] - with open(path) as f: - lines = f.readlines() - for i, line in enumerate(lines): - if f"{job_id}:" in line: - line_number = i + 1 - - lint_message = LintMessage( - path=str(path), - line=line_number, - char=None, - code="WORKFLOWSYNC", - severity=LintSeverity.ERROR, - name="workflow-inconsistency", - original=None, - replacement=None, - description=f"Job doesn't match other jobs with sync-tag: '{sync_tag}'", - ) - print(json.dumps(lint_message._asdict()), flush=True) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="workflow consistency linter.", - fromfile_prefix_chars="@", - ) - parser.add_argument( - "filenames", - nargs="+", - help="paths to lint", - ) - args = parser.parse_args() - - # Go through the provided files, aggregating jobs with the same sync tag - tag_to_jobs = defaultdict(list) - for path in args.filenames: - workflow = load_yaml(Path(path)) - jobs = workflow["jobs"] - for job_id, job in jobs.items(): - try: - sync_tag = job["with"]["sync-tag"] - except KeyError: - continue - - # remove the "if" field, which we allow to be different between jobs - # (since you might have different triggering conditions on pull vs. - # trunk, say.) - if "if" in job: - del job["if"] - - tag_to_jobs[sync_tag].append((path, {job_id: job})) - - # For each sync tag, check that all the jobs have the same code. - for sync_tag, path_and_jobs in tag_to_jobs.items(): - baseline_path, baseline_dict = path_and_jobs.pop() - baseline_str = dump(baseline_dict) - - printed_baseline = False - - for path, job_dict in path_and_jobs: - job_str = dump(job_dict) - if baseline_str != job_str: - print_lint_message(path, job_dict, sync_tag) - - if not printed_baseline: - print_lint_message(baseline_path, baseline_dict, sync_tag) - printed_baseline = True