From 604a44ea7965c8af74539ff2c3a27f1ce62f8c8f Mon Sep 17 00:00:00 2001 From: alingse Date: Tue, 24 Sep 2019 21:09:27 +0800 Subject: [PATCH 1/5] see github tags --- ChangeLog.md | 29 ----------------------------- 1 file changed, 29 deletions(-) delete mode 100644 ChangeLog.md diff --git a/ChangeLog.md b/ChangeLog.md deleted file mode 100644 index e584c5c..0000000 --- a/ChangeLog.md +++ /dev/null @@ -1,29 +0,0 @@ -2019.06.12 - -- 2.1.0 add convert_json && dump_excel to use as library - -2018.05.26 - -- 2.0.9 support python3 - -2018.05.04 - -- 2.0.8 support unicode - -2018.05.03 - -- 2.0.8b fix setup on windows(maybe) - -2018.04.09 - -- 2.0.8a clean code & add travis - -2018.03.30 - -- 2.0.7 support sort headers on dump csv - -2017.05.10 - -- 2.0.6 clean the mkexcel - -....... the earlier version is jsut... fun From 201252bb70ded259c9df259e23d34ff9a778d3fa Mon Sep 17 00:00:00 2001 From: alingse Date: Tue, 24 Sep 2019 21:11:13 +0800 Subject: [PATCH 2/5] try support json array --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 9ed40eb..46135ff 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,7 @@ python: - "2.7" - "3.5" - "3.6" + - "3.7" install: - pip install . - pip install flake8 From 45a8fb4f36bb8670ecff025461ed5399f06741cc Mon Sep 17 00:00:00 2001 From: alingse Date: Tue, 24 Sep 2019 23:05:34 +0800 Subject: [PATCH 3/5] add -A / --array to support json array --- jsoncsv/jsontool.py | 24 ++++++++++++++++++++---- jsoncsv/main.py | 11 +++++++++-- setup.py | 10 ++++++---- 3 files changed, 35 insertions(+), 10 deletions(-) diff --git a/jsoncsv/jsontool.py b/jsoncsv/jsontool.py index 5765e97..88be8a1 100755 --- a/jsoncsv/jsontool.py +++ b/jsoncsv/jsontool.py @@ -134,17 +134,33 @@ def restore(expobj, separator='.', safe=False): return origin -def convert_json(fin, fout, func, separator=".", safe=False): +def convert_json(fin, fout, func, separator=".", safe=False, json_array=False): if func not in [expand, restore]: raise ValueError("unknow convert_json type") - for line in fin: - obj = json.loads(line) + # default: read json objects from each line + def gen_objs(): + for line in fin: + obj = json.loads(line) + yield obj + + objs = gen_objs() + + if json_array: + # read all input as json array + def gen_objs_from_array(): + objs = json.load(fin) + assert isinstance(objs, list) + for obj in objs: + yield obj + + objs = gen_objs_from_array() + + for obj in objs: new = func(obj, separator=separator, safe=safe) content = json.dumps(new, ensure_ascii=False) if PY2: fout.write(content.encode('utf-8')) else: fout.write(content) - fout.write(str('\n')) diff --git a/jsoncsv/main.py b/jsoncsv/main.py index 63191a9..46b63c2 100644 --- a/jsoncsv/main.py +++ b/jsoncsv/main.py @@ -11,6 +11,13 @@ @click.command() +@click.option( + '-A', + '--array', + 'json_array', + is_flag=True, + default=False, + help='read input file as json array') @click.option( '-s', '--sep', @@ -42,7 +49,7 @@ 'output', type=click.File('w'), default=sys.stdout) -def jsoncsv(output, input, expand, restore, safe, separator): +def jsoncsv(output, input, expand, restore, safe, separator, json_array): if expand and restore: raise click.UsageError('can not choose both, default is `-e`') @@ -50,7 +57,7 @@ def jsoncsv(output, input, expand, restore, safe, separator): if restore: func = jsontool.restore - convert_json(input, output, func, separator, safe) + convert_json(input, output, func, separator=separator, safe=safe, json_array=json_array) input.close() output.close() diff --git a/setup.py b/setup.py index ad134d9..6af9cb5 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ setup( name='jsoncsv', - version='2.2.1', + version='2.2.2', url='https://github.com/alingse/jsoncsv', description='A command tool easily convert json file to csv or xlsx.', long_description=readme, @@ -28,6 +28,7 @@ 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', ], install_requires=[ 'unicodecsv', @@ -42,11 +43,12 @@ }, keywords=[ 'jsontocsv', + 'json2csv', 'jsoncsv', + 'command', + 'convert', 'json', 'csv', - 'xls' - 'convert', - 'command', + 'xls', ], ) From 651e1dfe81ab34d8e220979bec066921476ac5c1 Mon Sep 17 00:00:00 2001 From: alingse Date: Tue, 24 Sep 2019 23:05:53 +0800 Subject: [PATCH 4/5] add test --- fixture/files/raw.1.json | 1 + tests/test_jsoncsv.py | 7 ++++++- tests/test_jsontool.py | 14 ++++++++++++++ 3 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 fixture/files/raw.1.json diff --git a/fixture/files/raw.1.json b/fixture/files/raw.1.json new file mode 100644 index 0000000..b73dc45 --- /dev/null +++ b/fixture/files/raw.1.json @@ -0,0 +1 @@ +[{"a":{"b":1}}, {"a":{"c":2}}] diff --git a/tests/test_jsoncsv.py b/tests/test_jsoncsv.py index 5c6c6de..2e4c241 100644 --- a/tests/test_jsoncsv.py +++ b/tests/test_jsoncsv.py @@ -14,5 +14,10 @@ def test_jsoncsv_expand(self): runner = CliRunner() args = ['-e', 'fixture/files/raw.0.json', 'fixture/files/tmp.expand.0.json'] result = runner.invoke(jsoncsv, args=args) - print(result) + assert result.exit_code == 0 + + def test_jsoncsv_expand_with_json_array(self): + runner = CliRunner() + args = ['-e', 'fixture/files/raw.1.json', 'fixture/files/tmp.expand.1.json', '-A'] + result = runner.invoke(jsoncsv, args=args) assert result.exit_code == 0 diff --git a/tests/test_jsontool.py b/tests/test_jsontool.py index 19f0a8f..1a87224 100644 --- a/tests/test_jsontool.py +++ b/tests/test_jsontool.py @@ -120,3 +120,17 @@ def test_convert_restore(self): fin.close() fout.close() + + def test_convert_expand_json_array(self): + fin = io.StringIO('[{"a":{"b":3}},{"a":{"c":4}}]') + if PY2: + fout = io.BytesIO() + else: + fout = io.StringIO() + + convert_json(fin, fout, expand, json_array=True) + + self.assertEqual('{"a.b": 3}\n{"a.c": 4}\n', fout.getvalue()) + + fin.close() + fout.close() From 54f6f47a492a21c643f93910a58a36eb026e92e0 Mon Sep 17 00:00:00 2001 From: alingse Date: Tue, 24 Sep 2019 23:07:34 +0800 Subject: [PATCH 5/5] add more test --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 46135ff..b1f467b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,9 @@ script: - cat fixture/files/raw.0.json|jsoncsv -e > fixture/files/tmp.expand.0.json - cat fixture/files/raw.0.json|jsoncsv -e|jsoncsv -r|jsoncsv > fixture/files/tmp.expand.0.json - cat fixture/files/expand.2.json|jsoncsv|mkexcel -t xls > fixture/files/tmp.output.2.xls - - cat fixture/files/expand.2.json|jsoncsv|mkexcel -t csv > fixture/files/tmp.output.2..csv + - cat fixture/files/expand.2.json|jsoncsv|mkexcel -t csv > fixture/files/tmp.output.2.csv + - cat fixture/files/raw.1.json|jsoncsv -A|mkexcel > fixture/files/tmp.output.1.csv + - cat fixture/files/raw.1.json|jsoncsv --array|mkexcel > fixture/files/tmp.output.1.csv after_success: - coveralls notifications: