From 571d360eb2d825bdceb79ab994cb83f07761121f Mon Sep 17 00:00:00 2001 From: "Malcolm Jones (bossjones/Tony Dark)" Date: Mon, 3 Sep 2018 01:23:47 -0400 Subject: [PATCH] Feature: feature-run-in-flatpak (#302) --- HACKING.md | 213 +++++++++ Makefile | 21 + RESEARCH.md | 437 ++++++++++++++++++ hacking/flatpak/generate-manifest.py | 367 +++++++++++++++ hacking/flatpak/org.scarlett.ScarlettOS.json | 20 + scarlett_os/bootstrap.py | 1 + scarlett_os/commands.py | 13 + scarlett_os/common/configure/default.yaml | 2 +- scarlett_os/common/configure/ruamel_config.py | 63 +-- scarlett_os/config.py | 1 + scarlett_os/const.py | 6 + scarlett_os/core.py | 2 +- scarlett_os/internal/debugger.py | 2 + scarlett_os/listener.py | 35 +- scarlett_os/player.py | 146 +++--- scarlett_os/speaker.py | 4 +- scarlett_os/subprocess.py | 5 +- scarlett_os/tasker.py | 35 +- scarlett_os/tools/package.py | 6 + scarlett_os/utility/observable.py | 63 +++ scarlett_os/utility/thread.py | 5 +- scripts/contrib/render_jhbuild.py | 8 + scripts/run_scarlett_system | 13 + scripts/run_scarlett_system_profiler | 13 + scripts/symlink-homebrew-site-packages | 76 +++ tests/integrationtests/baseclass.py | 82 ++++ .../test_integration_player.py | 8 +- .../test_integration_tasker.py | 5 + .../common/configure/test_ruamel_config.py | 77 +++ tests/unittests/test_player.py | 4 +- tests/unittests/test_tasker.py | 4 +- 31 files changed, 1614 insertions(+), 123 deletions(-) create mode 100644 HACKING.md create mode 100755 hacking/flatpak/generate-manifest.py create mode 100644 scarlett_os/utility/observable.py create mode 100755 scripts/run_scarlett_system create mode 100755 scripts/run_scarlett_system_profiler create mode 100755 scripts/symlink-homebrew-site-packages diff --git a/HACKING.md b/HACKING.md new file mode 100644 index 00000000..f2047e6d --- /dev/null +++ b/HACKING.md @@ -0,0 +1,213 @@ +--- +short-description: Using the ScarlettOS development environment +... + +# Hacking on ScarlettOS + +## The easy way + +The easy way to setup the development environment is to follow the +[GNOME Newcomers guide](https://wiki.gnome.org/Newcomers/). + +Make sure to use the right git repository: + +> **https://gitlab.gnome.org/GNOME/pitivi.git** + + +## Setting up the advanced development environment + +> NOTE: This way of setting the development environment is sensibly more complex +> but also more flexible than the one for newcomers. If you are a beginner +> or if you usually use [gnome-builder](https://wiki.gnome.org/Apps/Builder) +> as your main IDE, follow, as previously adviced, the +> [GNOME Newcomers guide](https://wiki.gnome.org/Newcomers/) + +The official way of getting your environment up and running is by using +[flatpak](http://flatpak.org/). For this you need to +[install flatpak](http://flatpak.org/getting.html) on your system, +along with flatpak-builder. Note flatpak-builder might be provided by an +additional package on some distributions (such as Archlinux). + +Create a development environment folder and get the [ScarlettOS source code](http://gitlab.gnome.org/GNOME/pitivi) into it: + +``` +$ mkdir pitivi-dev +$ cd pitivi-dev +$ git clone https://gitlab.gnome.org/GNOME/pitivi.git +``` + +Whenever you want to hack on ScarlettOS, enter the development environment: +``` +$ cd pitivi-dev/pitivi && source bin/pitivi-env +-> Setting up the prefix for the sandbox... +Using ScarlettOS prefix in /.../pitivi-dev/pitivi-prefix +[prefix being built, if not already...] +Running in sandbox: echo Prefix ready +Prefix ready +``` + +This can take a while when creating the sandbox from scratch. Note the +prompt changes: +``` +(ptv-flatpak) $ +``` + +By entering the development environment, you get: +- a [Flatpak sandbox](http://docs.flatpak.org/en/latest/working-with-the-sandbox.html) +for the dependencies, in `pitivi-dev/pitivi-prefix` +- a [Python virtual environment](http://docs.python-guide.org/en/latest/dev/virtualenvs/) +with development tools, such as +[pre-commit](http://pre-commit.com), +in `pitivi-dev/pitivi/build/flatpak/pyvenv` +- the [Meson build directory](http://mesonbuild.com/Quick-guide.html), +in `pitivi-dev/pitivi/mesonbuild` +- some aliases for the build tools, such as `ninja`, so they are executed in the sandbox. + +Now that you are in the development environment, try running the +[unittests](Testing.md): +``` +(ptv-flatpak) $ ninja -C mesonbuild/ test +Running in sandbox: ninja -C mesonbuild/ test +``` + +Hack away, and check the effect of your changes by simply running: +``` +(ptv-flatpak) $ pitivi +``` + + +### Updating the development environment + +To update the dependencies installed in the sandbox, run: +``` +(ptv-flatpak) $ ptvenv --update +``` + +That will actually recreate the sandbox prefix, updating all +dependencies from their git repos and tarballs as defined in the +[flatpak +manifest](https://git.gnome.org/browse/pitivi/tree/build/flatpak/pitivi.template.json) (located at `build/flatpak/pitivi.template.json`). + + +### How we use the sandbox + +The sandbox we set up has access to the host file system. This allows +running the Python modules in `pitivi-dev/pitivi/pitivi/...` on top of +the GNOME + ScarlettOS dependencies system installed in the sandbox. +Without this trick, you'd have to build and install every time when you +change a `.py` file, to be able to test how it works, which would be +annoying because it takes a non-negligible amount of time. + +We don't actually run ScarlettOS 100% uninstalled. Besides the `.py` files +there are other parts which need to be built when changed or even +installed before using them: + +- Select parts of ScarlettOS are written in C, such as the audio envelope +renderer for the audio clips. Build them with `ninja -C mesonbuild/` or +with our very own alias `build`, which is the same thing. No need to +install them. + +- Similarly, `bin/pitivi.py.in` and `pitivi/configure.py.in` also need +to be built with `build`, to regenerate the corresponding `.py` files. + +- The translations need to be built and installed, which can be done +with `binstall`. See "Switching locales" below. + + +### Hacking on ScarlettOS dependencies (Meson) + +If you have to work on say, [GStreamer Editing Services](https://gstreamer.freedesktop.org/modules/gst-editing-services.html) +which is built using the Meson build system, first clone it into your +`pitivi-dev` folder: +``` +(ptv-flatpak) $ git clone git://anongit.freedesktop.org/gstreamer/gst-editing-services +``` + +Prepare its build directory. Once it has been set up, you won't have to +run `meson` again for this build directory. +``` +(ptv-flatpak) $ setup +Using ScarlettOS prefix in /.../pitivi-dev/pitivi-prefix +Running in sandbox: meson mesonbuild/ --prefix=/app --libdir=lib -Ddisable_gtkdoc=true -Ddisable_doc=true +``` + +Build and install it in the sandbox: +``` +(ptv-flatpak) $ ninja -C mesonbuild/ install +Using ScarlettOS prefix in /.../pitivi-dev/pitivi-prefix +Running in sandbox: ninja -C mesonbuild/ install +``` + +In the `(ptv-flatpak)` development environment `meson` and `ninja` are +aliases which run meson and ninja in the flatpak sandbox. + +NOTE: When updating the environment with `ptvenv --update`, +it will use your local dependencies repositories it finds in the +`pitivi-dev` folder, instead of the default remote repositories. +This means you have to update them yourself. +Also beware that it will not take into account not committed +changes. + + +### Hacking on ScarlettOS dependencies (Autotools, Make, etc) + +If the project you are working on is built with other tools, make sure +they are run in the sandbox by using `ptvenv`. For example: + +``` +(ptv-flatpak) $ cd pitivi-dev/frei0r-plugins-1.4 +(ptv-flatpak) $ ptvenv ./autogen.sh +Running in sandbox: ./autogen.sh +(ptv-flatpak) $ ptvenv ./configure +Running in sandbox: ./configure +(ptv-flatpak) $ ptvenv make +Running in sandbox: make +``` + + +## Profiling ScarlettOS + +To profile a ScarlettOS run, simply set the `PITIVI_PROFILING` environment +variable to 1, like so: + +``` +(ptv-flatpak) $ PITIVI_PROFILING=1 pitivi +``` + +A file named `pitivi-runstats` will be created in the current directory, a handy tool to examine it is `gprof2dot.py`, install it with: + +``` +$ pip install gprof2dot +``` + +Then run: + +``` +$ gprof2dot -f pstats pitivi-runstats | dot -Tsvg -o profile.svg +``` + +You can then inspect the call tree profile with your preferred image viewer: + +``` +$ xdg-open profile.svg +``` + + +## Switching locales + +To see how ScarlettOS looks in a different locale, use: + +``` +(ptv-flatpak) $ LANG=fr_FR.UTF-8 pitivi +``` + +Pay attention the translations in the sandbox are not automatically +updated when you `git pull`. You can update them by updating your +sandbox (`ptvenv --update`) or by reinstalling ScarlettOS in the sandbox: + +``` +(ptv-flatpak) $ binstall +[...] +Installing /.../pitivi-dev/pitivi/mesonbuild/po/de.gmo to /app/share/locale/de/LC_MESSAGES/pitivi.mo +[...] +``` diff --git a/Makefile b/Makefile index c6e8e13f..29028f62 100644 --- a/Makefile +++ b/Makefile @@ -900,7 +900,17 @@ makelint-install: # @echo 'git commit -a -m "Released $(RELEASE) via make release"' # @echo 'git tag --force v$(VERSION)' # @echo 'git push --tags origin master' +.PHONY: jhbuild-profile-tasker +jhbuild-profile-tasker: + jhbuild run python -m cProfile -s cumulative scarlett_os/tasker.py +.PHONY: jhbuild-profile-mpris +jhbuild-profile-mpris: + jhbuild run python -m cProfile -s cumulative scarlett_os/mpris.py + +.PHONY: jhbuild-profile-listener +jhbuild-profile-listener: + jhbuild run python -m cProfile -s cumulative scarlett_os/listener.py # NOTE: You can also run pylint with warnings turned into errors using python -W error -m pylint … to get a traceback for the warnings. # SOURCE: https://github.com/neomake/neomake/issues/1828#issuecomment-377901357 @@ -912,6 +922,10 @@ run-pylint-error: jhbuild-run-pylint-error: jhbuild run -- pylint -E scarlett_os +.PHONY: jhbuild-install +jhbuild-install: + jhbuild run python setup.py install + .PHONY: jhbuild-run-pylint-warning-stacktrace jhbuild-run-pylint-warning-stacktrace: jhbuild run -- python -W error -m pylint -E scarlett_os @@ -1107,3 +1121,10 @@ ninja-install-uninstalled: ninja-build -C mesonbuild/ install meson-install-uninstalled: meson-build-uninstalled ninja-install-uninstalled + +# Since we don't want to write to homebrew system python directly, create a virtualenv them add modules in via symlinks +# MUST MATCH SAME VERSION! +pyenv-virtualenv37: + virtualenv -p /usr/local/bin/python3 --system-site-packages ~/.pyenv/versions/system37 + +# ~/.pyenv/versions/system37/lib/python3.7/site-packages \ No newline at end of file diff --git a/RESEARCH.md b/RESEARCH.md index 577a9c01..74a7eca0 100644 --- a/RESEARCH.md +++ b/RESEARCH.md @@ -2353,3 +2353,440 @@ os.environ['CFLAGS'] = '-fPIC -O0 -ggdb -fno-inline -fno-omit-frame-pointer' os.environ['PYTHON'] = '/usr/bin/python3' os.environ['PROMPT_COMMAND'] = '' ``` + + +# GstPocketSphinx object. dump() + +``` +obj.__class__ = +obj.__copy__ = +obj.__deepcopy__ = +obj.__delattr__ = +obj.__dict__ = {} +obj.__dir__ = +obj.__doc__ = Object GstPocketSphinx + +Properties from GstPocketSphinx: + hmm -> gchararray: HMM Directory + Directory containing acoustic model parameters + lm -> gchararray: LM File + Language model file + lmctl -> gchararray: LM Control File + Language model control file (for class LMs) + dict -> gchararray: Dictionary File + Dictionary File + fsg -> gchararray: FSG File + Finite state grammar file + fwdflat -> gboolean: Flat Lexicon Search + Enable Flat Lexicon Search + bestpath -> gboolean: Graph Search + Enable Graph Search + maxhmmpf -> gint: Maximum HMMs per frame + Maximum number of HMMs searched per frame + maxwpf -> gint: Maximum words per frame + Maximum number of words searched per frame + beam -> gdouble: Beam width applied to every frame in Viterbi search + Beam width applied to every frame in Viterbi search + wbeam -> gdouble: Beam width applied to word exits + Beam width applied to phone transitions + pbeam -> gdouble: Beam width applied to phone transitions + Beam width applied to phone transitions + dsratio -> gint: Frame downsampling ratio + Evaluate acoustic model every N frames + latdir -> gchararray: Lattice Directory + Output Directory for Lattices + lmname -> gchararray: LM Name + Language model name (to select LMs from lmctl) + decoder -> PSDecoder: Decoder object + The underlying decoder + +Signals from GstElement: + pad-added (GstPad) + pad-removed (GstPad) + no-more-pads () + +Signals from GstObject: + deep-notify (GstObject, GParam) + +Properties from GstObject: + name -> gchararray: Name + The name of the object + parent -> GstObject: Parent + The parent of the object + +Signals from GObject: + notify (GParam) + + +obj.__eq__ = +obj.__format__ = +obj.__gdoc__ = Object GstPocketSphinx + +Properties from GstPocketSphinx: + hmm -> gchararray: HMM Directory + Directory containing acoustic model parameters + lm -> gchararray: LM File + Language model file + lmctl -> gchararray: LM Control File + Language model control file (for class LMs) + dict -> gchararray: Dictionary File + Dictionary File + fsg -> gchararray: FSG File + Finite state grammar file + fwdflat -> gboolean: Flat Lexicon Search + Enable Flat Lexicon Search + bestpath -> gboolean: Graph Search + Enable Graph Search + maxhmmpf -> gint: Maximum HMMs per frame + Maximum number of HMMs searched per frame + maxwpf -> gint: Maximum words per frame + Maximum number of words searched per frame + beam -> gdouble: Beam width applied to every frame in Viterbi search + Beam width applied to every frame in Viterbi search + wbeam -> gdouble: Beam width applied to word exits + Beam width applied to phone transitions + pbeam -> gdouble: Beam width applied to phone transitions + Beam width applied to phone transitions + dsratio -> gint: Frame downsampling ratio + Evaluate acoustic model every N frames + latdir -> gchararray: Lattice Directory + Output Directory for Lattices + lmname -> gchararray: LM Name + Language model name (to select LMs from lmctl) + decoder -> PSDecoder: Decoder object + The underlying decoder + +Signals from GstElement: + pad-added (GstPad) + pad-removed (GstPad) + no-more-pads () + +Signals from GstObject: + deep-notify (GstObject, GParam) + +Properties from GstObject: + name -> gchararray: Name + The name of the object + parent -> GstObject: Parent + The parent of the object + +Signals from GObject: + notify (GParam) + + +obj.__ge__ = +obj.__getattribute__ = +obj.__gpointer__ = +obj.__grefcount__ = 2 +obj.__gsignals__ = {} +obj.__gt__ = +obj.__gtype__ = +obj.__hash__ = +obj.__info__ = ObjectInfo(Element) +obj.__init__ = +obj.__le__ = +obj.__lt__ = +obj.__module__ = __gi__ +obj.__ne__ = +obj.__new__ = +obj.__reduce__ = +obj.__reduce_ex__ = +obj.__repr__ = +obj.__setattr__ = +obj.__sizeof__ = +obj.__str__ = +obj.__subclasshook__ = +obj._force_floating = gi.FunctionInfo(force_floating) +obj._gst_reserved = [0, 0, 0] +obj._ref = gi.FunctionInfo(ref) +obj._ref_sink = gi.FunctionInfo(ref_sink) +obj._unref = gi.FunctionInfo(unref) +obj._unsupported_data_method = > +obj._unsupported_method = > +obj.abort_state = gi.FunctionInfo(abort_state) +obj.add_control_binding = gi.FunctionInfo(add_control_binding) +obj.add_metadata = > +obj.add_pad = gi.FunctionInfo(add_pad) +obj.add_pad_template = > +obj.add_static_metadata = > +obj.add_static_pad_template = > +obj.base_time = 0 +obj.bind_property = +obj.bind_property_full = > +obj.bus = +obj.chain = +obj.change_state = gi.FunctionInfo(change_state) +obj.check_uniqueness = gi.FunctionInfo(check_uniqueness) +obj.clock = None +obj.compat_control = > +obj.connect = +obj.connect_after = +obj.connect_data = > +obj.connect_object = +obj.connect_object_after = +obj.contexts = [] +obj.continue_state = gi.FunctionInfo(continue_state) +obj.control_bindings = [] +obj.control_rate = 100000000 +obj.create_all_pads = gi.FunctionInfo(create_all_pads) +obj.current_state = +obj.default_deep_notify = gi.FunctionInfo(default_deep_notify) +obj.default_error = gi.FunctionInfo(default_error) +obj.disconnect = .meth of <__gi__.GstPocketSphinx object at 0x7efe628dd8b8 (GstPocketSphinx at 0x7efe54010150)>> +obj.disconnect_by_func = +obj.do_change_state = gi.VFuncInfo(change_state) +obj.do_deep_notify = gi.VFuncInfo(deep_notify) +obj.do_get_state = gi.VFuncInfo(get_state) +obj.do_no_more_pads = gi.VFuncInfo(no_more_pads) +obj.do_pad_added = gi.VFuncInfo(pad_added) +obj.do_pad_removed = gi.VFuncInfo(pad_removed) +obj.do_post_message = gi.VFuncInfo(post_message) +obj.do_provide_clock = gi.VFuncInfo(provide_clock) +obj.do_query = gi.VFuncInfo(query) +obj.do_release_pad = gi.VFuncInfo(release_pad) +obj.do_request_new_pad = gi.VFuncInfo(request_new_pad) +obj.do_send_event = gi.VFuncInfo(send_event) +obj.do_set_bus = gi.VFuncInfo(set_bus) +obj.do_set_clock = gi.VFuncInfo(set_clock) +obj.do_set_context = gi.VFuncInfo(set_context) +obj.do_set_state = gi.VFuncInfo(set_state) +obj.do_state_changed = gi.VFuncInfo(state_changed) +obj.emit = +obj.emit_stop_by_name = > +obj.find_property = > +obj.flags = 0 +obj.force_floating = > +obj.freeze_notify = > +obj.g_type_instance = +obj.get_base_time = gi.FunctionInfo(get_base_time) +obj.get_bus = gi.FunctionInfo(get_bus) +obj.get_clock = gi.FunctionInfo(get_clock) +obj.get_compatible_pad = gi.FunctionInfo(get_compatible_pad) +obj.get_compatible_pad_template = gi.FunctionInfo(get_compatible_pad_template) +obj.get_context = gi.FunctionInfo(get_context) +obj.get_context_unlocked = gi.FunctionInfo(get_context_unlocked) +obj.get_contexts = gi.FunctionInfo(get_contexts) +obj.get_control_binding = gi.FunctionInfo(get_control_binding) +obj.get_control_rate = gi.FunctionInfo(get_control_rate) +obj.get_data = > +obj.get_factory = gi.FunctionInfo(get_factory) +obj.get_g_value_array = gi.FunctionInfo(get_g_value_array) +obj.get_metadata = > +obj.get_name = gi.FunctionInfo(get_name) +obj.get_pad_template = > +obj.get_pad_template_list = > +obj.get_parent = gi.FunctionInfo(get_parent) +obj.get_path_string = gi.FunctionInfo(get_path_string) +obj.get_properties = +obj.get_property = +obj.get_qdata = > +obj.get_request_pad = gi.FunctionInfo(get_request_pad) +obj.get_start_time = gi.FunctionInfo(get_start_time) +obj.get_state = gi.FunctionInfo(get_state) +obj.get_static_pad = gi.FunctionInfo(get_static_pad) +obj.get_value = gi.FunctionInfo(get_value) +obj.get_value_array = gi.FunctionInfo(get_value_array) +obj.handler_block = > +obj.handler_block_by_func = +obj.handler_disconnect = .meth of <__gi__.GstPocketSphinx object at 0x7efe628dd8b8 (GstPocketSphinx at 0x7efe54010150)>> +obj.handler_is_connected = .meth of <__gi__.GstPocketSphinx object at 0x7efe628dd8b8 (GstPocketSphinx at 0x7efe54010150)>> +obj.handler_unblock = .meth of <__gi__.GstPocketSphinx object at 0x7efe628dd8b8 (GstPocketSphinx at 0x7efe54010150)>> +obj.handler_unblock_by_func = +obj.has_active_control_bindings = gi.FunctionInfo(has_active_control_bindings) +obj.has_ancestor = gi.FunctionInfo(has_ancestor) +obj.has_as_ancestor = gi.FunctionInfo(has_as_ancestor) +obj.has_as_parent = gi.FunctionInfo(has_as_parent) +obj.install_properties = > +obj.install_property = > +obj.interface_find_property = > +obj.interface_install_property = > +obj.interface_list_properties = > +obj.is_floating = gi.FunctionInfo(is_floating) +obj.is_locked_state = gi.FunctionInfo(is_locked_state) +obj.iterate_pads = gi.FunctionInfo(iterate_pads) +obj.iterate_sink_pads = gi.FunctionInfo(iterate_sink_pads) +obj.iterate_src_pads = gi.FunctionInfo(iterate_src_pads) +obj.last_return = +obj.last_sync = 18446744073709551615 +obj.link = gi.FunctionInfo(link) +obj.link_filtered = gi.FunctionInfo(link_filtered) +obj.link_pads = gi.FunctionInfo(link_pads) +obj.link_pads_filtered = gi.FunctionInfo(link_pads_filtered) +obj.link_pads_full = gi.FunctionInfo(link_pads_full) +obj.list_properties = > +2018-08-08 23:43:47,099 scarlett_os.utility.gnome (Thread #748) ERROR (wrapper) Exception Thrown from [/home/pi/dev/bossjones-github/scarlett_os/scarlett_os/internal/debugger.py] on line [162] via function [dump] +2018-08-08 23:43:47,099 scarlett_os.utility.gnome (Thread #748) ERROR (wrapper) Exception type NotImplementedError: getting an union is not supported yet +emit ('aborted', (, NotImplementedError('getting an union is not supported yet',), )) +``` + +# pocketsphinx_continuous + +``` + +/home/pi/dev/bossjones-github/scarlett_os/scarlett_os/listener.py:202: PyGIDeprecationWarning: GObject.MainLoop is deprecated; use GLib.MainLoop instead + self.__loop = GObject.MainLoop() +Running +Current configuration: +[NAME] [DEFLT] [VALUE] +-agc none none +-agcthresh 2.0 2.000000e+00 +-allphone +-allphone_ci no no +-alpha 0.97 9.700000e-01 +-ascale 20.0 2.000000e+01 +-aw 1 1 +-backtrace no no +-beam 1e-48 1.000000e-48 +-bestpath yes yes +-bestpathlw 9.5 9.500000e+00 +-ceplen 13 13 +-cmn current current +-cmninit 8.0 40,3,-1 +-compallsen no no +-debug 0 +-dict /home/pi/dev/bossjones-github/scarlett_os/static/speech/dict/1473.dic +-dictcase no no +-dither no no +-doublebw no no +-ds 1 1 +-fdict +-feat 1s_c_d_dd 1s_c_d_dd +-featparams +-fillprob 1e-8 1.000000e-08 +-frate 100 100 +-fsg +-fsgusealtpron yes yes +-fsgusefiller yes yes +-fwdflat yes yes +-fwdflatbeam 1e-64 1.000000e-64 +-fwdflatefwid 4 4 +-fwdflatlw 8.5 8.500000e+00 +-fwdflatsfwin 25 25 +-fwdflatwbeam 7e-29 7.000000e-29 +-fwdtree yes yes +-hmm /home/pi/.virtualenvs/scarlett_os/share/pocketsphinx/model/en-us/en-us +-input_endian little little +-jsgf +-keyphrase +-kws +-kws_delay 10 10 +-kws_plp 1e-1 1.000000e-01 +-kws_threshold 1 1.000000e+00 +-latsize 5000 5000 +-lda +-ldadim 0 0 +-lifter 0 22 +-lm /home/pi/dev/bossjones-github/scarlett_os/static/speech/lm/1473.lm +-lmctl +-lmname +-logbase 1.0001 1.000100e+00 +-logfn +-logspec no no +-lowerf 133.33334 1.300000e+02 +-lpbeam 1e-40 1.000000e-40 +-lponlybeam 7e-29 7.000000e-29 +-lw 6.5 6.500000e+00 +-maxhmmpf 30000 3000 +-maxwpf -1 -1 +-mdef +-mean +-mfclogdir +-min_endfr 0 0 +-mixw +-mixwfloor 0.0000001 1.000000e-07 +-mllr +-mmap yes yes +-ncep 13 13 +-nfft 512 512 +-nfilt 40 25 +-nwpen 1.0 1.000000e+00 +-pbeam 1e-48 1.000000e-48 +-pip 1.0 1.000000e+00 +-pl_beam 1e-10 1.000000e-10 +-pl_pbeam 1e-10 1.000000e-10 +-pl_pip 1.0 1.000000e+00 +-pl_weight 3.0 3.000000e+00 +-pl_window 5 5 +-rawlogdir +-remove_dc no no +-remove_noise yes yes +-remove_silence yes yes +-round_filters yes yes +-samprate 16000 1.600000e+04 +-seed -1 -1 +-sendump +-senlogdir +-senmgau +-silprob 0.005 5.000000e-03 +-smoothspec no no +-svspec 0-12/13-25/26-38 +-tmat +-tmatfloor 0.0001 1.000000e-04 +-topn 4 4 +-topn_beam 0 0 +-toprule +-transform legacy dct +-unit_area yes yes +-upperf 6855.4976 6.800000e+03 +-uw 1.0 1.000000e+00 +-vad_postspeech 50 50 +-vad_prespeech 20 20 +-vad_startspeech 10 10 +-vad_threshold 2.0 2.000000e+00 +-var +-varfloor 0.0001 1.000000e-04 +-varnorm no no +-verbose no no +-warp_params +-warp_type inverse_linear inverse_linear +-wbeam 7e-29 7.000000e-29 +-wip 0.65 6.500000e-01 +-wlen 0.025625 2.562500e-02 + +``` + +# Pocketsphinx how do I make it fast + +``` +How do I make it fast? + +The default settings are not enough to achieve sub-realtime performance on most tasks. Here are some command-line flags you should experiment with: + +-beam: Beam width applied to every frame in Viterbi search (smaller values mean wider beam)" + +-pbeam: Beam width applied to phone transitions + +-wbeam: Beam width applied to word exits + +Main parameters to configure search width and thus accuracy-performance balance. + +-ds + +This is the dsratio. In most cases -ds 2 gives the best performance, though accuracy suffers a bit. (Frame GMM computation downsampling ratio) Thus lower should be better and higher should be less accurate. + +-topn + +The default value is 4, the fastest value is 2, but accuracy can suffer a bit depending on your acoustic model. + +-lpbeam + +This beam is quite important for performance, however the default setting is pretty narrow already. Run pocketsphinx_batch with no arguments to see what it is. + +-lponlybeam + +Likewise here as with -lpbeam. If you are finding it hard to get enough accuracy, you can widen these beams. + +-maxwpf + +This can be set quite low and still give you reasonable performance - try 5. + +-maxhmmpf + +Depending on the acoustic and language model this can be very helpful. Try 3000. + +-pl_window + +Phonetic lookahead is a specific technique which is used to speedup decoding by reducing the amount of computation. Basically everything is decoded with phonetic decoder first and then detailed search is restricted by the results of the fast phonetic search. It’s also called “Fast match”. For details and evaluations see the chapter “4.5 Phonetic Fast Match” in Efficient Algorithms for Speech Recognition Mosur K. Ravishankar + +pl_window specifies lookahead distance in frames. Typical values are from 0 (don’t use lookahead) to 10 (decode 10 frames ahead). Bigger values give faster decoding but reduced accuracy. +``` diff --git a/hacking/flatpak/generate-manifest.py b/hacking/flatpak/generate-manifest.py new file mode 100755 index 00000000..c935bc39 --- /dev/null +++ b/hacking/flatpak/generate-manifest.py @@ -0,0 +1,367 @@ +#!/usr/bin/env python3 + +import json as jsonlib +import requests +import os +import hashlib +import argparse + +# SOURCE: https://github.com/AdrianKoshka/flatpak-tools/blob/master/org.mozilla.Thunderbird/genman.py + +# Setup arguments to be parsed +parser = argparse.ArgumentParser(description="Auto generates ScarlettOS' flatpak manifest") +parser.add_argument("-r", "--release", help="ScarlettOS release version") +parser.add_argument("-o", "--output", help="File to write to", default="org.scarlett.ScarlettOS.updated.json") +args = parser.parse_args() + +# File to output the JSON to +output_file = args.output + +# Version of the GNOME runtime to use +gnome_runtime = "3.24" +python_vers = "3.5.2" + +# Take the thunderbird release from the '-r' or --release' argument +release = args.release + +# A function which takes a URL, requests the content, and makes a sha256 hash +# of it, and then returns said hash +def hashsrc(url): + print("Getting " + url) + r = requests.get(url) + sha256 = hashlib.sha256() + sha256.update(r.content) + filechecksum = sha256.hexdigest() + return(filechecksum) + + +fin_args = [ + # / * Allow access to developer tools * / + "--allow=devel", + "--talk-name=org.freedesktop.Flatpak", + # / * X11 + XShm access * / + "--socket=x11", + "--share=ipc", + # /* OpenGL */ + "--device=dri", + # /* Wayland access */ + "--socket=wayland", + # /* Audio output */ + "--socket=pulseaudio", + + # /* We want full fs access */ + "--filesystem=host", + "--filesystem=home", + + + # /* Allow communication with network */ + "--share=network", + "--talk-name=org.gtk.vfs.*", + + # /* Needed for dconf to work (+ host or homedir read access from above) */ + "--filesystem=xdg-run/dconf", + "--filesystem=~/.config/dconf:ro", + "--talk-name=ca.desrt.dconf", + "--env=DCONF_USER_CONFIG_DIR=.config/dconf", + + # /* We need access to auth agents */ + "--talk-name=org.freedesktop.secrets", + "--filesystem=xdg-run/keyring", + + # / * Needed for various SSL certificates to work * / + "--env=SSL_CERT_DIR=/etc/ssl/certs", + "--filesystem=/etc/ssl:ro", + "--filesystem=/etc/pki:ro", + "--filesystem=/etc/ca-certificates:ro", + + # / * Keep system terminal mappings * / + "--filesystem=/etc/inputrc:ro", + + # / * Chromium uses a socket in tmp for its singleton check * / + "--filesystem=/tmp", + + "--own-name=org.scarlett.Listener", + "--own-name=org.scarlett.Listener.*", + + # Bunch of dbus session bus stuff + "--talk-name=org.freedesktop.DBus.Proprieties", + "--talk-name=org.freedesktop.IBus", + "--talk-name=org.freedesktop.Notifications", + + # Applications sometimes need to interact with the desktop's file manager. + # SOURCE: https://www.freedesktop.org/wiki/Specifications/file-manager-interface/ + "--talk-name=org.freedesktop.FileManager1", + + # Gnome settings daemon + "--talk-name=org.gnome.SettingsDaemon.Color", + "--talk-name=org.freedesktop.PackageKit", + + # Ability to talk to polkit + "--system-talk-name=org.freedesktop.PolicyKit1", + + # Sysprof kernel based performance profiler for Linux + # SOURCE: https://github.com/GNOME/sysprof + "--system-talk-name=org.gnome.Sysprof2", + + # gnome-code-assistance is a project which aims to provide common code assistance + # services for code editors(simple editors as well as IDEs). It is an effort to + # provide a centralized code - assistance as a service for the GNOME platform + # instead of having every editor implement their own solution. + + # SOURCE: https://github.com/GNOME/gnome-code-assistance + "--talk-name=org.gnome.CodeAssist.v1.*", + + "--system-talk-name=org.freedesktop.login1", + + "--socket=session-bus", + "--system-talk-name=org.freedesktop.Avahi", + + "--filesystem=~/.local/share/flatpak", + "--filesystem=/var/lib/flatpak", + "--filesystem=xdg-data/meson" + +] + +# # Define the finish-args +# fin_args = [ +# "--share=ipc", +# "--socket=x11", +# "--device=dri", +# "--share=network", +# "--socket=pulseaudio", +# "--filesystem=~/.cache/thunderbird:create", +# "--filesystem=~/.thunderbird:create", +# "--filesystem=home:ro", +# "--filesystem=xdg-download:rw", +# "--filesystem=xdg-run/dconf", +# "--filesystem=~/.config/dconf:ro", +# "--talk-name=ca.desrt.dconf", +# "--env=DCONF_USER_CONFIG_DIR=.config/dconf", +# "--talk-name=org.a11y.*", +# "--talk-name=org.freedesktop.Notifications" +# ] + +# Define the files/directories to cleanup +# clnup = [ +# "/include", +# "/lib/pkgconfig", +# "/share/pkgconfig", +# "/share/aclocal", +# "/man", +# "/share/man", +# "*.la", +# "*.a" +# ] +clnup = [] + +# Define the structure for the build-options section +build_opts = {} +build_opts["cflags"] = "-O2 -g" +build_opts["cxxflags"] = "-O2 -g" +build_opts["env"] = { + "V": "1", + "BASH_COMPLETIONSDIR": "/app/share/bash-completion/completions", + "MOUNT_FUSE_PATH": "../tmp/" +} + +# Define the modules section +mdles = [] + +######################################################### +# Autoconf sources +######################################################### +acsrcs = [] +acsrc = {} +acsrc["type"] = "archive" +acsrc["url"] = "http://ftp.gnu.org/gnu/autoconf/autoconf-2.13.tar.gz" +acsrc["sha256"] = hashsrc(acsrc["url"]) +acsrcs.append(acsrc) + +# Autoconf module +ac = {} +ac["name"] = "autoconf-2.13" +# ac["cleanup"] = ["*"] +ac["sources"] = acsrcs +ac["post-install"] = ["ln -s /app/bin/autoconf /app/bin/autoconf-2.13"] +mdles.append(ac) +######################################################### + + + +######################################################### +# icu sources +######################################################### +icusrcs = [] +icusrc = {} +icusrc["type"] = "archive" +icusrc["url"] = "http://download.icu-project.org/files/icu4c/60.1/icu4c-60_1-src.tgz" +icusrc["sha256"] = hashsrc(icusrc["url"]) +icusrcs.append(icusrc) + +# icu module +icu = {} +icu["name"] = "icu" +icu["cleanup"] = ["/bin/*", "/sbin/*"] +icu["sources"] = icusrc +icu["subdir"] = {"subdir": "source"} +mdles.append(icu) +######################################################### + + +######################################################### +# cpython sources +######################################################### +cpython_sources = [] +cpythonsrc = {} +cpythonsrc["type"] = "archive" +cpythonsrc["url"] = "https://www.python.org/ftp/python/${python_vers}/Python-${python_vers}.tar.xz".format(python_vers=python_vers) +cpythonsrc["sha256"] = hashsrc(cpythonsrc["url"]) +cpython_sources.append(cpythonsrc) + +# pip source +pipsrc = {} +pipsrc["type"] = "file" +pipsrc["url"] = "https://files.pythonhosted.org/packages/ae/e8/2340d46ecadb1692a1e455f13f75e596d4eab3d11a57446f08259dee8f02/pip-10.0.1.tar.gz" +pipsrc["sha256"] = hashsrc(pipsrc["url"]) +cpython_sources.append(pipsrc) +######################################################### + +######################################################### +# setuptools source +######################################################### +stsrc = {} +stsrc["type"] = "file" +stsrc["url"] = "https://files.pythonhosted.org/packages/a6/5b/f399fcffb9128d642387133dc3aa9bb81f127b949cd4d9f63e5602ad1d71/setuptools-39.1.0.zip" +stsrc["sha256"] = hashsrc(stsrc["url"]) +cpython_sources.append(stsrc) + +# wheel source +wheelsrc = {} +wheelsrc["type"] = "file" +wheelsrc["url"] = "https://files.pythonhosted.org/packages/5d/c1/45947333669b31bc6b4933308dd07c2aa2fedcec0a95b14eedae993bd449/wheel-0.31.0.tar.gz" +wheelsrc["sha256"] = hashsrc(wheelsrc["url"]) +cpython_sources.append(wheelsrc) + +# cpython module +cpython = {} +cpython["name"] = "cpython" +# cpython["cleanup"] = ["*"] +cpython["sources"] = cpython_sources +cpython["post-install"] = [ + "ls -lta `pwd`", + "/app/bin/python3 -m pip install --no-index --find-links=\"file://${PWD}\" --prefix=${FLATPAK_DEST} pip setuptools wheel" +] +cpython["build-options"] = { + "build-args": [ + "--share=network", + "--allow=devel" + ], + "config-opts": [ + "--with-pydebug" + ], + "cflags": "-O0 -g", + "cxxflags": "-O0 -g", + "strip": False, + "no-debuginfo": False +} + +mdles.append(cpython) +######################################################### + + +# Thunderbird build-options +# app_data_and_icons_bopt = {} +# app_data_and_icons_bopt["clfags"] = "-fno-delete-null-pointer-checks -fno-lifetime-dse -fno-schedule-insns2" +# app_data_and_icons_bopt["cxxflags"] = "-fno-delete-null-pointer-checks -fno-lifetime-dse -fno-schedule-insns2" +# app_data_and_icons_bopt["env"] = {"VERSION": release} + +# Thunderbird build-commands +app_data_and_icons_bc = [ + "ls -lta", + "env", + "mkdir -p /app/share/metainfo/", + "mkdir -p /app/share/appdata/", + "mkdir -p /app/share/applications/", + "mkdir -p /app/cache/scarlett/", + "mkdir -p /app/share/icons/hicolor/64x64/apps/", + "cp org.scarlett.ScarlettOS.appdata.xml /app/share/metainfo/org.scarlett.ScarlettOS.appdata.xml", + "cp scarlettOS.png /app/share/icons/hicolor/64x64/apps/org.scarlett.ScarlettOS.png", + "cp org.scarlett.ScarlettOS.desktop /app/share/applications/org.scarlett.ScarlettOS.desktop" +] + +# Thunderbird sources +appdata_and_icons_src = [] + +# mozconfig source +icons_install_src = {} +icons_install_src["type"] = "file" +icons_install_src["path"] = "../../data/icons-install.sh" +icons_install_src["dest-filename"] = "icons-install.sh" +appdata_and_icons_src.append(icons_install_src) + +# .desktop file +dsk_install_shell_script = {} +dsk_install_shell_script["type"] = "file" +dsk_install_shell_script["path"] = "../../data/install-desktop-file.sh" +dsk_install_shell_script["dest-filename"] = "install-desktop-file.sh" +appdata_and_icons_src.append(dsk_install_shell_script) + +# AppData source +png_data = {} +png_data["type"] = "file" +png_data["path"] = "../../data/icons/64x64/scarlettOS.png" +appdata_and_icons_src.append(png_data) + +# AppData source +desktop_data = {} +desktop_data["type"] = "file" +desktop_data["path"] = "../../data/ScarlettOS.desktop" +desktop_data["dest-filename"] = "org.scarlett.ScarlettOS.desktop" +appdata_and_icons_src.append(desktop_data) + +appdata_xml = {} +appdata_xml["type"] = "file" +appdata_xml["path"] = "../../data/ScarlettOS.appdata.xml" +appdata_xml["dest-filename"] = "org.scarlett.ScarlettOS.appdata.xml" +appdata_and_icons_src.append(appdata_xml) + +# URL formation +# burl = "https://ftp.mozilla.org/pub/thunderbird/releases/" +# srcdir = "/source/" +# srctar = "thunderbird-" + release + ".source.tar.xz" +# full_url = burl + release + srcdir + srctar + +# Thunderbird source tar +# tbtarsrc = {} +# tbtarsrc["type"] = "archive" +# tbtarsrc["url"] = full_url +# tbtarsrc["sha256"] = hashsrc(tbtarsrc["url"]) +# appdata_and_icons_src.append(tbtarsrc) + +# appdata_and_icons module +appdata_and_icons_module = {} +appdata_and_icons_module["name"] = "appdata_and_icons" +appdata_and_icons_module["buildsystem"] = "simple" +# appdata_and_icons_module["build-options"] = app_data_and_icons_bopt +appdata_and_icons_module["build-commands"] = app_data_and_icons_bc +appdata_and_icons_module["sources"] = appdata_and_icons_src +mdles.append(appdata_and_icons_module) + +# Define the basic structure +base = {} +base["app-id"] = "org.scarlett.ScarlettOS" +base["runtime"] = gnome_runtime +base["sdk"] = "org.gnome.Sdk" +base["command"] = "/usr/bin/bash" +base["tags"] = ["nightly"] +base["finish-args"] = fin_args +base["build-options"] = build_opts +base["cleanup"] = clnup +base["modules"] = mdles +base["desktop-file-name-prefix"] = "(Nightly) " +base["copy-icon"] = True +json_data = jsonlib.dumps(base, indent=4, default=str) + +# Spit out the JSON +with open(output_file, 'w') as f: + f.write(json_data) diff --git a/hacking/flatpak/org.scarlett.ScarlettOS.json b/hacking/flatpak/org.scarlett.ScarlettOS.json index e154c5a9..1236e89f 100644 --- a/hacking/flatpak/org.scarlett.ScarlettOS.json +++ b/hacking/flatpak/org.scarlett.ScarlettOS.json @@ -1085,6 +1085,26 @@ ] }, + { + "name": "scarlettOS", + "buildsystem": "meson", + "builddir": true, + "config-opts": [ + "--libdir=lib" + ], + "build-options": { + "build-args": [ + "--share=network" + ] + }, + "sources": [ + { + "type": "dir", + "url": "../../" + } + ] + }, + { "name": "appdata_and_icons", "buildsystem": "simple", diff --git a/scarlett_os/bootstrap.py b/scarlett_os/bootstrap.py index bb562c6e..d74e6dcb 100644 --- a/scarlett_os/bootstrap.py +++ b/scarlett_os/bootstrap.py @@ -25,6 +25,7 @@ import scarlett_os.core as core from scarlett_os.exceptions import ScarlettError + # from scarlett_os.utility.yaml import load_yaml import scarlett_os.helpers.config_validation as cv from scarlett_os.helpers.entity import set_customize diff --git a/scarlett_os/commands.py b/scarlett_os/commands.py index 40aeb0de..4509d30e 100644 --- a/scarlett_os/commands.py +++ b/scarlett_os/commands.py @@ -134,6 +134,7 @@ } NO_OP = "__SCARLETT_NO_OP__" +RETRY_OP = "__SCARLETT_RETRY_OP__" ###################################################################################################################### # NOTE: In the future time will be its own plugin class etc, for now we're just going to add arbitrary functions here: @@ -176,12 +177,14 @@ def check_cmd(command_tuple=None): command, SPOTIFY_CMDS[command] ) ) + return "spotify command" elif command in LIGHT_CMDS.keys(): logger.debug( "** received {}, sending 'light {}'".format( command, LIGHT_CMDS[command] ) ) + return "light command" elif command in TIME_CMDS.keys(): logger.debug( "** received {}, sending 'time {}'".format(command, TIME_CMDS[command]) @@ -194,15 +197,25 @@ def check_cmd(command_tuple=None): command, GENERAL_CMDS[command] ) ) + return "general command" elif command in FORECAST_CMDS.keys(): logger.debug( "** received {}, sending 'forecast command: {}'".format( command, FORECAST_CMDS[command] ) ) + return "forecast command" elif command in TV_CMDS.keys(): logger.debug( "** received {}, sending 'tv command: {}'".format( command, TV_CMDS[command] ) ) + return "tv command" + else: + logger.debug( + "** received '{}', doesn't match anything, lets retry".format( + command + ) + ) + return RETRY_OP diff --git a/scarlett_os/common/configure/default.yaml b/scarlett_os/common/configure/default.yaml index d8b8e504..2cd6c309 100644 --- a/scarlett_os/common/configure/default.yaml +++ b/scarlett_os/common/configure/default.yaml @@ -18,7 +18,7 @@ pocketsphinx: # Enable Graph Search | Boolean. Default: true # ******************************************************** # Word insertion penalty - wip: 0.0001 + wip: 1e-4 device: plughw:CARD=Device,DEV=0 # ******************************************************** # FIXME: ????? THIS IS THE ORIG VALUE, do we need too set it back? 8/5/2018 # bestpath: 0 diff --git a/scarlett_os/common/configure/ruamel_config.py b/scarlett_os/common/configure/ruamel_config.py index 66a2b4b4..84004c2c 100644 --- a/scarlett_os/common/configure/ruamel_config.py +++ b/scarlett_os/common/configure/ruamel_config.py @@ -621,34 +621,6 @@ def get_version_file_path(override=None): ######################################################### -def _fake_config(override=None): - """Create a temporary config file.""" - base = tempfile.mkdtemp() - logger.debug("base tempfile: {}".format(base)) - config_file = os.path.join(base, "config.yaml") - - ############################################################# - # Example of config_file: - ############################################################# - # ⌁ pi@scarlett-ansible-manual1604-2 ~ ll /tmp/tmpnxz2wsa2 - # total 12 - # drwx------ 2 pi pi 4096 Feb 24 15:58 ./ - # drwxrwxrwt 15 root root 4096 Feb 24 15:58 ../ - # -rw-rw-r-- 1 pi pi 1034 Feb 24 15:58 config.yaml - # ⌁ pi@scarlett-ansible-manual1604-2 ~ - ############################################################# - - if override is not None: - with open(config_file, "wt") as f: - f.write(override) - return base, config_file - - with open(config_file, "wt") as f: - f.write(DEFAULT_CONFIG) - - return base, config_file - - def load_config(yaml_filename): """Load a yaml file into memory using ruamel.yaml.round_trip_load Arguments: @@ -1117,19 +1089,21 @@ def as_yaml_str(self): def prep_default_config(self): """setup config.yaml defaults.""" + self._config_path = ConfigManager.CONFIG_PATH + # Step 1. ensure sub directory actually exists self.check_folder_structure() # Step 2. check if config file exists, if it doesnt, create a default config - if not os.path.exists(ConfigManager.CONFIG_PATH): + if not os.path.exists(self._config_path): # Step 2a. Load default default_config = load_config(ConfigManager.DEFAULT_CONFIG) - save_config(default_config, ConfigManager.CONFIG_PATH) + save_config(default_config, self._config_path) print( "Default config is set, please don't forget to update your github tokens, webhook tokens, and jenkins configurations appropiately! Location = {}".format( - ConfigManager.CONFIG_PATH + self._config_path ) ) @@ -1168,6 +1142,33 @@ def prep_default_config(self): from scarlett_os.internal.debugger import dump # pylint: disable=W0611 from scarlett_os.internal.debugger import pprint_color # pylint: disable=W0611 + def _fake_config(override=None): + """Create a temporary config file.""" + base = tempfile.mkdtemp() + logger.debug("base tempfile: {}".format(base)) + config_file = os.path.join(base, "config.yaml") + + ############################################################# + # Example of config_file: + ############################################################# + # ⌁ pi@scarlett-ansible-manual1604-2 ~ ll /tmp/tmpnxz2wsa2 + # total 12 + # drwx------ 2 pi pi 4096 Feb 24 15:58 ./ + # drwxrwxrwt 15 root root 4096 Feb 24 15:58 ../ + # -rw-rw-r-- 1 pi pi 1034 Feb 24 15:58 config.yaml + # ⌁ pi@scarlett-ansible-manual1604-2 ~ + ############################################################# + + if override is not None: + with open(config_file, "wt") as f: + f.write(override) + return base, config_file + + with open(config_file, "wt") as f: + f.write(DEFAULT_CONFIG) + + return base, config_file + fake_config_file_path_base, fake_config_file_path = _fake_config( override=RUSSIAN_CONFIG ) diff --git a/scarlett_os/config.py b/scarlett_os/config.py index af848972..241fb3a3 100644 --- a/scarlett_os/config.py +++ b/scarlett_os/config.py @@ -27,6 +27,7 @@ ) from scarlett_os.core import valid_entity_id from scarlett_os.exceptions import ScarlettError + # from scarlett_os.utility.yaml import load_yaml import scarlett_os.helpers.config_validation as cv from scarlett_os.helpers.entity import set_customize diff --git a/scarlett_os/const.py b/scarlett_os/const.py index e198ec85..bdab421e 100644 --- a/scarlett_os/const.py +++ b/scarlett_os/const.py @@ -429,3 +429,9 @@ TEMPERATURE = "temperature" # type: str SPEED_MS = "speed_ms" # type: str ILLUMINANCE = "illuminance" # type: str + + +# PyGObject +# If the function returns False it is automatically removed from the list of event sources and will not be called again. +G_REMOVE_EVENT = False +G_CALLBACK_STOP = False diff --git a/scarlett_os/core.py b/scarlett_os/core.py index f297964a..703b9aad 100644 --- a/scarlett_os/core.py +++ b/scarlett_os/core.py @@ -5,7 +5,7 @@ of entities and react to changes. """ # pylint: disable=unused-import, too-many-lines -from concurrent.futures import ThreadPoolExecutor +# from concurrent.futures import ThreadPoolExecutor import enum import logging import os diff --git a/scarlett_os/internal/debugger.py b/scarlett_os/internal/debugger.py index 10801696..bf4f3944 100644 --- a/scarlett_os/internal/debugger.py +++ b/scarlett_os/internal/debugger.py @@ -7,6 +7,8 @@ logger = logging.getLogger(__name__) +SCARLETT_DEBUG_MODE = os.environ.get("SCARLETT_DEBUG_MODE") + # SOURCE: https://github.com/mjumbewu/jokosher/blob/e181d738674a98242b10dd697a9628be54c3121a/bin/jokosher # if platform.system() == "Windows": # ENV_PATHS = {"JOKOSHER_DATA_PATH" : ".\\", diff --git a/scarlett_os/listener.py b/scarlett_os/listener.py index 4635dc16..7e9883a5 100644 --- a/scarlett_os/listener.py +++ b/scarlett_os/listener.py @@ -158,6 +158,11 @@ } +# from pocketsphinx.pocketsphinx import * +# from sphinxbase.sphinxbase import * +# config = Decoder.default_config() +# config.set_float("-vad_threshold", 3.0) + ################################################################# # Managing the Gobject main loop thread. ################################################################# @@ -330,6 +335,8 @@ class ouside the dbus server. # __dr = None __instance = None + MAX_FAILURES = 10 + def __init__(self, name, config_manager, *args): threading.Thread.__init__(self) _IdleObject.__init__(self) @@ -460,7 +467,11 @@ def play(self): ret = p.set_state(Gst.State.PLAYING) if ret == Gst.StateChangeReturn.FAILURE: logger.error("ERROR: Unable to set the pipeline to the playing state") - self.on_debug_activate() + + # 8/8/2018 (Only enable this if we turn on debug mode) + if os.environ.get("SCARLETT_DEBUG_MODE"): + self.on_debug_activate() + logger.debug("BEFORE: self.ready_sem.acquire()") self.ready_sem.acquire() logger.debug("AFTER: self.ready_sem.acquire()") @@ -597,8 +608,6 @@ def on_debug_activate(self): if os.access(pngfile, os.F_OK): os.remove(pngfile) - # FIXME: This needs to use dynamic paths, it's possible that we're having issues because of order of operations - # FIXME: STATIC PATH 7/3/2018 Gst.debug_bin_to_dot_file( self.pipelines_stack[0], Gst.DebugGraphDetails.ALL, "generator-listener" ) @@ -623,6 +632,7 @@ def result(self, final_hyp): failed_temp = self.failed + 1 self.failed = failed_temp logger.debug("self.failed = {}".format(int(self.failed))) + # failed > 10 if self.failed > 4: # reset pipline self.dbus_proxy.emitSttFailedSignal() # CHANGEME @@ -695,6 +705,11 @@ def init_gst(self): # get gst pipeline element pocketsphinx and set properties - BEGIN # ************************************************************ pocketsphinx = pipeline.get_by_name("asr") + # from scarlett_os.internal.debugger import dump + # print("debug-2018-pocketsphinx - BEGIN") + # dump(pocketsphinx.get_property('decoder')) + # print("debug-2018-pocketsphinx - END") + # print(pocketsphinx.list_properties()) if self._hmm: pocketsphinx.set_property("hmm", self._hmm) if self._lm: @@ -717,11 +732,11 @@ def init_gst(self): if self._bestpath: pocketsphinx.set_property("bestpath", self._bestpath) - if self._silprob: - pocketsphinx.set_property("silprob", self._silprob) + # if self._silprob: + # pocketsphinx.set_property("silprob", self._silprob) - if self._wip: - pocketsphinx.set_property("wip", self._wip) + # if self._wip: + # pocketsphinx.set_property("wip", self._wip) # ************************************************************ # get gst pipeline element pocketsphinx and set properties - END # ************************************************************ @@ -859,6 +874,7 @@ def _new_sample(self, sink): # IMPORTANT!!!!! # NOTE: I think this is causing a deadlock self.queue.put(buf.extract_dup(0, buf.get_size())) + # "OK = 0. Data passing was ok."" return Gst.FlowReturn.OK def _on_message(self, bus, message): @@ -881,8 +897,11 @@ def _on_message(self, bus, message): self.read_exc = NoStreamError() self.ready_sem.release() elif struct and struct.get_name() == "pocketsphinx": + # "final", G_TYPE_BOOLEAN, final, + # SOURCE: https://github.com/cmusphinx/pocketsphinx/blob/1fdc9ccb637836d45d40956e745477a2bd3b470a/src/gst-plugin/gstpocketsphinx.c if struct["final"]: - logger.info(struct["hypothesis"]) + logger.info("confidence: {}".format(struct["confidence"])) + logger.info("hypothesis: {}".format(struct["hypothesis"])) if self.kw_found == 1: # If keyword is set AND qualifier # then perform action diff --git a/scarlett_os/player.py b/scarlett_os/player.py index efe4e730..53b3501f 100644 --- a/scarlett_os/player.py +++ b/scarlett_os/player.py @@ -58,6 +58,7 @@ from scarlett_os.internal.path import touch_empty_file from scarlett_os.common.configure.ruamel_config import ConfigManager +from scarlett_os.internal.debugger import dump # from scarlett_os.logger import setup_logger # @@ -104,7 +105,9 @@ def get_loop_thread(): class MainLoopThread(threading.Thread): """A daemon thread encapsulating a Gobject main loop. - A mainloop is used by all asynchronous objects to defer handlers and callbacks to. The function run() blocks until the function quit() has been called (and all queued handlers have been executed). The run() function will then execute all the handlers in order. + A mainloop is used by all asynchronous objects to defer handlers and callbacks to. + The function run() blocks until the function quit() has been called (and all queued handlers have been executed). + The run() function will then execute all the handlers in order. """ def __init__(self): @@ -134,6 +137,7 @@ def __init__(self, path, handle_error, callback): self.callback = callback # self.delay_start = False if delay_start is None else delay_start # self.stopme = threading.Event() + self._lock = threading.Lock() # Set up the Gstreamer pipeline. # NOTE: Need to add .new for some reason to pass tests ... @@ -299,7 +303,10 @@ def __init__(self, path, handle_error, callback): # return self.pipeline.set_state(Gst.State.PLAYING) - self.on_debug_activate() + # 8/8/2018 (Only enable this if we turn on debug mode) + if os.environ.get("SCARLETT_DEBUG_MODE"): + self.on_debug_activate() + self.ready_sem.acquire() if self.read_exc: @@ -315,43 +322,45 @@ def _source_setup_cb(self, discoverer, source): logger.debug("Source object: ({})".format(source)) def _on_state_changed(self, bus, msg): - states = msg.parse_state_changed() - # To state is PLAYING - if msg.src.get_name() == "pipeline" and states[1] == 4: - # TODO: Modify the creation of this path, it should be programatically created - # FIXME: This needs to use dynamic paths, it's possible that we're having issues because of order of operations - # FIXME: STATIC PATH 7/3/2018 - dotfile = ( - "/home/pi/dev/bossjones-github/scarlett_os/_debug/generator-player.dot" - ) - pngfile = "/home/pi/dev/bossjones-github/scarlett_os/_debug/generator-player-pipeline.png" # NOQA - - parent_dir = get_parent_dir(dotfile) - - mkdir_if_does_not_exist(parent_dir) - - if os.access(dotfile, os.F_OK): - os.remove(dotfile) - if os.access(pngfile, os.F_OK): - os.remove(pngfile) - - if not fname_exists(dotfile): - touch_empty_file(dotfile) - - if not fname_exists(pngfile): - touch_empty_file(pngfile) - - # FIXME: This needs to use dynamic paths, it's possible that we're having issues because of order of operations - # FIXME: STATIC PATH 7/3/2018 - Gst.debug_bin_to_dot_file( - msg.src, Gst.DebugGraphDetails.ALL, "generator-player" - ) - - cmd = "/usr/bin/dot -Tpng -o {pngfile} {dotfile}".format( - pngfile=pngfile, dotfile=dotfile - ) - os.system(cmd) - print("pipeline dot file created in " + os.getenv("GST_DEBUG_DUMP_DOT_DIR")) + # states = msg.parse_state_changed() + # # To state is PLAYING + # if msg.src.get_name() == "pipeline" and states[1] == 4: + # # NOTE: For right now, lets not try to write anything 8/8/2018 + # # TODO: Modify the creation of this path, it should be programatically created + # # FIXME: This needs to use dynamic paths, it's possible that we're having issues because of order of operations + # # FIXME: STATIC PATH 7/3/2018 + # dotfile = ( + # "/home/pi/dev/bossjones-github/scarlett_os/_debug/generator-player.dot" + # ) + # pngfile = "/home/pi/dev/bossjones-github/scarlett_os/_debug/generator-player-pipeline.png" # NOQA + + # parent_dir = get_parent_dir(dotfile) + + # mkdir_if_does_not_exist(parent_dir) + + # if os.access(dotfile, os.F_OK): + # os.remove(dotfile) + # if os.access(pngfile, os.F_OK): + # os.remove(pngfile) + + # if not fname_exists(dotfile): + # touch_empty_file(dotfile) + + # if not fname_exists(pngfile): + # touch_empty_file(pngfile) + + # # FIXME: This needs to use dynamic paths, it's possible that we're having issues because of order of operations + # # FIXME: STATIC PATH 7/3/2018 + # Gst.debug_bin_to_dot_file( + # msg.src, Gst.DebugGraphDetails.ALL, "generator-player" + # ) + + # cmd = "/usr/bin/dot -Tpng -o {pngfile} {dotfile}".format( + # pngfile=pngfile, dotfile=dotfile + # ) + # os.system(cmd) + # print("pipeline dot file created in " + os.getenv("GST_DEBUG_DUMP_DOT_DIR")) + pass def _listElements(self, bin, level=0): try: @@ -384,35 +393,36 @@ def _iteratePads(self, element): # then finally generates a png file, which it then displays def on_debug_activate(self): # FIXME: STATIC PATH 7/3/2018 - dotfile = ( - "/home/pi/dev/bossjones-github/scarlett_os/_debug/generator-player.dot" - ) - pngfile = "/home/pi/dev/bossjones-github/scarlett_os/_debug/generator-player-pipeline.png" # NOQA + # dotfile = ( + # "/home/pi/dev/bossjones-github/scarlett_os/_debug/generator-player.dot" + # ) + # pngfile = "/home/pi/dev/bossjones-github/scarlett_os/_debug/generator-player-pipeline.png" # NOQA - parent_dir = get_parent_dir(dotfile) + # parent_dir = get_parent_dir(dotfile) - mkdir_if_does_not_exist(parent_dir) + # mkdir_if_does_not_exist(parent_dir) - if os.access(dotfile, os.F_OK): - os.remove(dotfile) - if os.access(pngfile, os.F_OK): - os.remove(pngfile) + # if os.access(dotfile, os.F_OK): + # os.remove(dotfile) + # if os.access(pngfile, os.F_OK): + # os.remove(pngfile) - if not fname_exists(dotfile): - touch_empty_file(dotfile) + # if not fname_exists(dotfile): + # touch_empty_file(dotfile) - if not fname_exists(pngfile): - touch_empty_file(pngfile) + # if not fname_exists(pngfile): + # touch_empty_file(pngfile) - # FIXME: This needs to use dynamic paths, it's possible that we're having issues because of order of operations - # FIXME: STATIC PATH 7/3/2018 - Gst.debug_bin_to_dot_file( - self.pipeline, Gst.DebugGraphDetails.ALL, "generator-player" - ) - cmd = "/usr/bin/dot -Tpng -o {pngfile} {dotfile}".format( - pngfile=pngfile, dotfile=dotfile - ) - os.system(cmd) + # # FIXME: This needs to use dynamic paths, it's possible that we're having issues because of order of operations + # # FIXME: STATIC PATH 7/3/2018 + # Gst.debug_bin_to_dot_file( + # self.pipeline, Gst.DebugGraphDetails.ALL, "generator-player" + # ) + # cmd = "/usr/bin/dot -Tpng -o {pngfile} {dotfile}".format( + # pngfile=pngfile, dotfile=dotfile + # ) + # os.system(cmd) + pass # Gstreamer callbacks. @@ -563,6 +573,18 @@ def close(self, force=False): # File "/home/pi/dev/bossjones-github/scarlett_os/scarlett_os/player.py", line 531, in close # self.pipeline # AttributeError: 'ScarlettPlayer' object has no attribute 'pipeline' + print("[2018-debugging]: dump(self) - BEGIN") + print(self) + # dump(self) + print("[2018-debugging]: dump(self) - END") + # FIXME: 9/2/2018 - + # We might need to unref our bus object. Calling too many times seems to cause problems. + # Hi, I got an error while calling audio_open() several times(with .mp3 files). Looks like it is due to accumulated refs of bus objects in GstAudioFile. I gave it a monkey patch and apparently it seems fine. However, since my files are not gst-encoded, I'm not sure whether this would cause any problem with actual gst decoding tasks. + # In my case, I inserted following lines in GstAudioFile.close(): + # bus = self.pipeline.get_bus() + # bus.unref() + # bus.remove_signal_watch() + # SOURCE: https://github.com/beetbox/audioread/issues/28 ( khlukekim on Jan 28, 2016 ) self.pipeline.get_bus().remove_signal_watch() # Stop reading the file. diff --git a/scarlett_os/speaker.py b/scarlett_os/speaker.py index 496212b7..d732757a 100644 --- a/scarlett_os/speaker.py +++ b/scarlett_os/speaker.py @@ -163,7 +163,5 @@ def __exit__(self, exc_type, exc_val, exc_tb): ] for scarlett_text in tts_list: with s_thread.time_logger("Scarlett Speaks"): - path_to_espeak_tmp_wav = os.path.join( - STATIC_SOUNDS_PATH, "espeak_tmp.wav" - ) + path_to_espeak_tmp_wav = os.path.join(STATIC_SOUNDS_PATH, "espeak_tmp.wav") ScarlettSpeaker(text_to_speak=scarlett_text, wavpath=path_to_espeak_tmp_wav) diff --git a/scarlett_os/subprocess.py b/scarlett_os/subprocess.py index bffd111f..5b7f63cc 100644 --- a/scarlett_os/subprocess.py +++ b/scarlett_os/subprocess.py @@ -38,7 +38,10 @@ class Subprocess(GObject.GObject): :type fork: `bool` """ - __gtype_name__ = "Subprocess" + + # FIXME: _gobject.type_register(cls, namespace.get('__gtype_name__')) + # RuntimeError: could not create new GType: Subprocess(subclass of GObject) + # __gtype_name__ = "Subprocess" __gsignals__ = { "exited": ( GObject.SignalFlags.RUN_LAST, diff --git a/scarlett_os/tasker.py b/scarlett_os/tasker.py index a5306b46..ad38cd84 100644 --- a/scarlett_os/tasker.py +++ b/scarlett_os/tasker.py @@ -89,9 +89,7 @@ def get_path(sound_type): @staticmethod def get_speaker_path(): - path_to_espeak_tmp_wav = os.path.join( - STATIC_SOUNDS_PATH, "espeak_tmp.wav" - ) + path_to_espeak_tmp_wav = os.path.join(STATIC_SOUNDS_PATH, "espeak_tmp.wav") return [path_to_espeak_tmp_wav] @@ -355,7 +353,7 @@ def print_keyword_args(**kwargs): def print_args(args): # pragma: no cover - for i, v in enumerate(args): + for _, v in enumerate(args): print("another arg through *arg : {}".format(v)) @@ -439,12 +437,13 @@ def speaker_generator_func(): # source: https://wiki.gnome.org/Projects/PyGObject/Threading # PyGObject: uses yield True to pass control to the main loop in regular intervals. yield True + # NOTE: This is where we keep breaking intgr tests print("scarlett_text in tts_list") print("[scarlett_text]: {}".format(scarlett_text)) print("type[scarlett_text]: {}".format(type(scarlett_text))) _wavepath = SoundType.get_speaker_path()[0] p = player.ScarlettPlayer(_wavepath, False, False) - logger.error("Duration: p.duration: {}".format(p.duration)) + print("Duration: p.duration: {}".format(p.duration)) while True: try: yield next(p) @@ -512,6 +511,7 @@ def on_signal_recieved(*args, **kwargs): # 2. Perform command print("args[4]") print(args[4]) + # FIXME: Here, maybe we should have a retry like 5 times before we decided to finally bail out 8/9/2018 command_run_results = commands.Command.check_cmd(command_tuple=args[4]) logger.debug("[command_run_results]: {}".format(command_run_results)) @@ -523,6 +523,17 @@ def on_signal_recieved(*args, **kwargs): # 4. Espeak gst plugin doesn't work, write sound to wav file call_espeak_subprocess(command_run_results) + # NOTE: Example of why we always get None back from Subprocess (8/9/2018) + # payload: (' ScarlettListener caught a command match', 'pi-response', 'GO') + # 2018-08-09 00:04:35,890 scarlett_os.commands (MainThread) INFO (check_cmd) Valid command_tuple: (' ScarlettListener caught a command match', 'pi-response', 'GO') + # 2018-08-09 00:04:35,890 __main__ (MainThread) DEBUG (on_signal_recieved) [command_run_results]: None + # 2018-08-09 00:04:35,890 __main__ (MainThread) INFO (call_espeak_subprocess) Running in call_espeak_subprocess + # cmd: ['espeak', '-p75', '-s175', '-g1', '-w', '/home/pi/dev/bossjones-github/scarlett_os/scarlett_os/data/sounds/espeak_tmp.wav', '-ven+f3', '. None .'] + # 2018-08-09 00:04:35,892 scarlett_os.subprocess (MainThread) DEBUG (check_command_type) Running Command: 'espeak -p75 -s175 -g1 -w /home/pi/dev/bossjones-github/scarlett_os/scar + # lett_os/data/sounds/espeak_tmp.wav -ven+f3 . None .' + # 2018-08-09 00:04:35,892 scarlett_os.subprocess (MainThread) DEBUG (__init__) command: ['espeak', '-p75', '-s175', '-g1', '-w', '/home/pi/dev/bossjones-github/scarlett_os/scarle + # tt_os/data/sounds/espeak_tmp.wav', '-ven+f3', '. None .'] + # # 4. Scarlett Speaks # tts_list = SpeakerType.speaker_to_array(command_run_results) # run_speaker_result = run_speaker(speaker_generator_func) @@ -560,6 +571,12 @@ def on_signal_recieved(*args, **kwargs): enable_remote_debugging() + if os.environ.get("SCARLETT_PROFILE_MODE"): + import cProfile + + pr = cProfile.Profile() + pr.enable() + from scarlett_os.logger import setup_logger setup_logger() @@ -617,9 +634,17 @@ def on_signal_recieved(*args, **kwargs): logger.warning("Remove this while testing manually") logger.warning("***********************************************") st.reset() + + if os.environ.get("SCARLETT_PROFILE_MODE"): + pr.disable() + pr.print_stats(sort='time') pass except: raise else: # Close into a ipython debug shell loop.run() + + if os.environ.get("SCARLETT_PROFILE_MODE"): + pr.disable() + pr.print_stats(sort='time') diff --git a/scarlett_os/tools/package.py b/scarlett_os/tools/package.py index 72cd5ed4..e4c4e65f 100644 --- a/scarlett_os/tools/package.py +++ b/scarlett_os/tools/package.py @@ -24,8 +24,14 @@ def get_sys_module(): return sys +def get_distutils_module(): + import distutils + + return distutils + def get_distutils_sysconfig_function_get_python_lib(): + distutils = get_distutils_module() from distutils.sysconfig import get_python_lib return get_python_lib diff --git a/scarlett_os/utility/observable.py b/scarlett_os/utility/observable.py new file mode 100644 index 00000000..be401277 --- /dev/null +++ b/scarlett_os/utility/observable.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python # NOQA +# -*- coding: utf-8 -*- + +# source: http://code.activestate.com/recipes/327082-pseudo-threads-with-generators-and-pygtkgnome-pyth/ + +# vim:sw=4:et: +""" +Scarlett Observable module. + +Attempt to implement the observable pattern to solve weird race conditions between +Listener, Tasker, Player, Speaker during keyword matches. + +https://sourcemaking.com/files/v2/content/patterns/Observer_example1.png +""" + +import threading +from scarlett_os.internal.gi import gi +from scarlett_os.internal.gi import GObject +from scarlett_os.internal.gi import GLib + +# TODO, Get this working with a KeyWord Match object + + +# SOURCE: https://github.com/ReapOmen/open-media/blob/1ac6be3e50e66df65ef37fa91e090739db041fb1/openmedia/observable/observable.py + +class Observable(object): + """Aka Subject, eg Auctioneer.""" + + def __init__(self): + self._observers = [] + + def add_observer(self, observer): + self._observers.append(observer) + + def notify_observers(self, info): + for obs in self._observers: + event = threading.Event() + GLib.idle_add(obs.update, event, info) + + +class Observer(object): + """Aka Bidder in a auction. This class watches the Auctioneer and reacts to events.""" + + def __init__(self): + pass + + def update(self, event, info): + return False + + +# class Counter(object): +# def __init__(self, start = 0): +# self.lock = threading.Lock() +# self.value = start +# def increment(self): +# logging.debug('Waiting for a lock') +# self.lock.acquire() +# try: +# logging.debug('Acquired a lock') +# self.value = self.value + 1 +# finally: +# logging.debug('Released a lock') +# self.lock.release() diff --git a/scarlett_os/utility/thread.py b/scarlett_os/utility/thread.py index 9cffa525..6dcec5ea 100644 --- a/scarlett_os/utility/thread.py +++ b/scarlett_os/utility/thread.py @@ -24,9 +24,10 @@ import contextlib import time -from scarlett_os.internal.debugger import init_debugger +if os.environ.get("SCARLETT_DEBUG_MODE"): + from scarlett_os.internal.debugger import init_debugger -init_debugger() + init_debugger() import signal import threading diff --git a/scripts/contrib/render_jhbuild.py b/scripts/contrib/render_jhbuild.py index aaa87b0b..f207e1e4 100755 --- a/scripts/contrib/render_jhbuild.py +++ b/scripts/contrib/render_jhbuild.py @@ -337,6 +337,7 @@ os.environ['CC'] = 'gcc' os.environ['PROJECT_HOME'] = '{PROJECT_HOME}' os.environ['PYTHONSTARTUP'] = '{PYTHONSTARTUP}' +os.environ['GI_TYPELIB_PATH'] = '{PREFIX}/lib/girepository-1.0' """ @@ -546,6 +547,13 @@ def compile_one(package_to_build): ) _popen_stdout(_rendered_command, cwd=path_to_folder) +def clean_one(package_to_build): + pkg_dict = get_package_dict(package_to_build) + path_to_folder = os.path.join(CHECKOUTROOT, pkg_dict[package_to_build]["folder"]) + with cd(path_to_folder): + _rendered_command = "jhbuild make clean" + _popen_stdout(_rendered_command, cwd=path_to_folder) + def compile_all(): compile_one("gtk-doc") diff --git a/scripts/run_scarlett_system b/scripts/run_scarlett_system new file mode 100755 index 00000000..71be49c4 --- /dev/null +++ b/scripts/run_scarlett_system @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Execute lint to spot code mistakes. + +cd "$(dirname "$0")/.." + +make run-mpris & +sleep 3 + +make run-tasker & +sleep 2 + +make run-listener & +sleep 3 diff --git a/scripts/run_scarlett_system_profiler b/scripts/run_scarlett_system_profiler new file mode 100755 index 00000000..cf4011b9 --- /dev/null +++ b/scripts/run_scarlett_system_profiler @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Execute lint to spot code mistakes. + +cd "$(dirname "$0")/.." + +make jhbuild-profile-mpris > mpris.prof.log 2>&1 & +sleep 3 + +make jhbuild-profile-tasker > tasker.prof.log 2>&1 & +sleep 2 + +make jhbuild-profile-listener > listener.prof.log 2>&1 & +sleep 3 diff --git a/scripts/symlink-homebrew-site-packages b/scripts/symlink-homebrew-site-packages new file mode 100755 index 00000000..1125bfad --- /dev/null +++ b/scripts/symlink-homebrew-site-packages @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +# Execute lint to spot code mistakes. + +# |2.4.3| using virtualenv: system37 dev7behance1484 in ~/.pyenv/versions/system37/lib/python3.7/site-packages +# ○ → ls -lta /usr/local/lib/python3.7/site-packages/ +# total 216 +# drwxr-xr-x 8 malcolm staff 256 Aug 16 21:32 virtualenv-16.0.0.dist-info +# drwxr-xr-x 7 malcolm staff 224 Aug 16 21:32 virtualenv_support +# drwxr-xr-x 23 malcolm staff 736 Aug 16 21:32 . +# drwxr-xr-x 5 malcolm staff 160 Aug 16 21:32 __pycache__ +# -rw-r--r-- 1 malcolm staff 99548 Aug 16 21:32 virtualenv.py +# drwxr-xr-x 5 malcolm staff 160 Aug 16 20:27 pygtkcompat +# lrwxr-xr-x 1 malcolm staff 95 Aug 16 20:27 pygobject-3.28.3-py3.7.egg-info -> ../../../Cellar/pygobject3/3.28.3_1/lib/python3.7/site-packages/pygobject-3.28.3-py3.7.egg-info +# drwxr-xr-x 18 malcolm staff 576 Aug 16 20:27 gi +# drwxr-xr-x 11 malcolm staff 352 Aug 1 10:50 pyside2uic +# drwxr-xr-x 8 malcolm staff 256 Aug 1 10:50 PySide2-5.11.0-py3.7.egg-info +# drwxr-xr-x 55 malcolm staff 1760 Aug 1 10:50 PySide2 +# drwxr-xr-x 6 malcolm staff 192 Aug 1 10:37 pycairo-1.17.1-py3.7.egg-info +# drwxr-xr-x 7 malcolm staff 224 Aug 1 10:37 cairo +# drwxr-xr-x 9 malcolm staff 288 Aug 1 08:41 wheel-0.31.1-py3.7.egg-info +# drwxr-xr-x 17 malcolm staff 544 Aug 1 08:41 wheel +# drwxr-xr-x 8 malcolm staff 256 Aug 1 08:41 pip-18.0-py3.7.egg-info +# drwxr-xr-x 7 malcolm staff 224 Aug 1 08:41 pip +# drwxr-xr-x 9 malcolm staff 288 Aug 1 08:41 setuptools-40.0.0-py3.7.egg-info +# drwxr-xr-x 7 malcolm staff 224 Aug 1 08:41 pkg_resources +# drwxr-xr-x 42 malcolm staff 1344 Aug 1 08:41 setuptools +# -rw-r--r-- 1 malcolm staff 1972 Aug 1 08:41 sitecustomize.py +# drwxr-xr-x 3 malcolm staff 96 Aug 1 08:41 .. +# -rw-r--r-- 1 malcolm staff 126 Jun 27 02:06 easy_install.py + +if [[ -z $PYTHON3 ]]; then + PY3_MINOR_VER=`/usr/local/bin/python3 --version 2>&1 | cut -d " " -f 2 | cut -d "." -f 2` + if (( $PY3_MINOR_VER < 5 )); then + echo "Couldn't find python3 in \$PATH that is >=3.5" + echo "Please install python3.5 or later or explicity define the python3 executable name with \$PYTHON3" + echo "Exiting here" + exit 1 + else + export PYTHON3="python3.$PY3_MINOR_VER" + export PYTHON3_VERSION=`/usr/local/bin/python3 --version 2>&1 | cut -d " " -f 2 | awk -F"." '{print $1"."$2}'` + export _HOMEBREW_SITE_PACKAGES="/usr/local/lib/python${PYTHON3_VERSION}/site-packages/" + export _HOMEBREW_PYENV_SITE_PACKAGES=`python -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())"` + echo "\$PY3_MINOR_VER=$PY3_MINOR_VER" + echo "\$PYTHON3=$PYTHON3" + echo "\$PYTHON3_VERSION=$PYTHON3_VERSION" + echo "\$_HOMEBREW_SITE_PACKAGES=$_HOMEBREW_SITE_PACKAGES" + fi +fi + +array=( 'pygtkcompat' 'pygobject*' 'gi' 'pycairo*' 'cairo' ) +for i in "${array[@]}" +do + # echo "ln -s ${_HOMEBREW_SITE_PACKAGES}${i} ${_HOMEBREW_PYENV_SITE_PACKAGES}/${i}" + # ln -s ${_HOMEBREW_SITE_PACKAGES}${i} ${_HOMEBREW_PYENV_SITE_PACKAGES}/${i} + echo "ls ${_HOMEBREW_SITE_PACKAGES}${i}" + echo -e "\n" + ls ${_HOMEBREW_SITE_PACKAGES}$i + # ln -s ${_HOMEBREW_SITE_PACKAGES}$i ${_HOMEBREW_PYENV_SITE_PACKAGES}/$i +done + +# ls -lta ${_HOMEBREW_SITE_PACKAGES} + + +# NOTE: This is what worked. AUTOMATE THIS AND WRITE A README FOR RUNNING SCARLETT BINDINGS AGAINST BLEEDING EDGE OSX/HOMEBREW +# virtualenv -p /usr/local/bin/python3 --system-site-packages ~/.pyenv/versions/system37 +# pyenv activate system37 +# pip install jupyter +# python -m ipykernel install --user +# pip install ptpython bpython +# cd `python -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())"` +# ln -sf /usr/local/lib/python3.7/site-packages/pygtkcompat +# ln -sf /usr/local/lib/python3.7/site-packages/pygobject-3.28.3-py3.7.egg-info +# ln -sf /usr/local/lib/python3.7/site-packages/gi +# ln -sf /usr/local/lib/python3.7/site-packages/pycairo-1.17.1-py3.7.egg-info +# ln -sf /usr/local/lib/python3.7/site-packages/cairo + diff --git a/tests/integrationtests/baseclass.py b/tests/integrationtests/baseclass.py index 991dde6e..52833411 100644 --- a/tests/integrationtests/baseclass.py +++ b/tests/integrationtests/baseclass.py @@ -7,11 +7,13 @@ """ import os +# import gc import sys import signal import pytest import builtins import threading +# import traceback import unittest import unittest.mock as mock @@ -33,6 +35,7 @@ from scarlett_os.internal.gi import Gio # noqa from scarlett_os.internal.gi import GObject # noqa from scarlett_os.internal.gi import GLib +# from scarlett_os.internal.gi import Gst from scarlett_os import tasker @@ -41,6 +44,34 @@ # from scarlett_os.utility.dbus_runner import DBusRunner +######################################################################################### +# NOTE: Are we ready to test for garbage collection leaks? +######################################################################################### +# def handle_uncaught_exception(exctype, value, trace): +# traceback.print_tb(trace) +# print(value, file=sys.stderr) +# sys.exit(1) + + +# sys.excepthook = handle_uncaught_exception + + +# def handle_glog(domain, level, message, udata): +# Gst.debug_print_stack_trace() +# traceback.print_stack() +# print("%s - %s" % (domain, message), file=sys.stderr) +# sys.exit(-11) + + +# # GStreamer Not enabled because of an assertion on caps on the CI server. +# # See https://gitlab.gnome.org/thiblahute/pitivi/-/jobs/66570 +# for category in ["Gtk", "Gdk", "GLib-GObject", "GES"]: +# GLib.log_set_handler(category, GLib.LogLevelFlags.LEVEL_CRITICAL, handle_glog, None) + +detect_leaks = os.environ.get("SCARLETT_TEST_DETECT_LEAKS", "0") not in ("0", "") +# os.environ["PITIVI_USER_CACHE_DIR"] = tempfile.mkdtemp(suffix="pitiviTestsuite") +######################################################################################### + def run_emitter_signal(request, get_environment, sig_name="ready"): print("Setting up emitter") @@ -84,7 +115,10 @@ def cb(pid, status): # @pytest.mark.usefixtures("service_on_outside", "get_environment", "get_bus") class IntegrationTestbase(object): + # _tracked_types = (Gst.MiniObject, Gst.Element, Gst.Pad, Gst.Caps) + """Base class for integration tests.""" + # _tracked_types = (Gst.MiniObject, Gst.Element, Gst.Pad, Gst.Caps) # Tests are not allowed to have an __init__ method # Python shells, the underscore (_) means the result of the last evaluated expression in the shell: @@ -102,6 +136,9 @@ def setup_method(self, _): self.status = None self.tasker = None + # if detect_leaks: + # self.gctrack() + def setup_tasker(self, monkeypatch, get_bus): """Create ScarlettTasker object and call setup_controller.""" monkeypatch.setattr( @@ -117,6 +154,51 @@ def teardown_method(self, _): self.tasker.reset() self.tasker = None + # if detect_leaks: + # self.gccollect() + # self.gcverify() + + # def gctrack(self): + # self.gccollect() + # self._tracked = [] + # for obj in gc.get_objects(): + # if not isinstance(obj, self._tracked_types): + # continue + + # self._tracked.append(obj) + + # def gccollect(self): + # ret = 0 + # while True: + # c = gc.collect() + # ret += c + # if c == 0: + # break + # return ret + + # def gcverify(self): + # leaked = [] + # for obj in gc.get_objects(): + # if not isinstance(obj, self._tracked_types) or \ + # obj in self._tracked: + # continue + + # leaked.append(obj) + + # # we collect again here to get rid of temporary objects created in the + # # above loop + # self.gccollect() + + # for elt in leaked: + # print(elt) + # for i in gc.get_referrers(elt): + # print(" ", i) + + # # NOTE: I think these guys are the same, not sure + # # self.assertFalse(leaked, leaked) + # assert not leaked, leaked + # del self._tracked + # @pytest.mark.usefixtures("service_on_outside", "get_environment", "get_bus") class IntegrationTestbaseMainloop(IntegrationTestbase): diff --git a/tests/integrationtests/test_integration_player.py b/tests/integrationtests/test_integration_player.py index 017e7f6f..505cabb5 100644 --- a/tests/integrationtests/test_integration_player.py +++ b/tests/integrationtests/test_integration_player.py @@ -45,12 +45,12 @@ def player_mocker_stopall(mocker): "Stop previous mocks, yield mocker plugin obj, then stopall mocks again" print("Called [setup]: mocker.stopall()") mocker.stopall() - print("Called [setup]: imp.reload(threadmanager)") + print("Called [setup]: imp.reload(player)") imp.reload(player) yield mocker print("Called [teardown]: mocker.stopall()") mocker.stopall() - print("Called [setup]: imp.reload(threadmanager)") + print("Called [setup]: imp.reload(player)") imp.reload(player) @@ -67,9 +67,7 @@ def test_ScarlettPlayer_listening(self, player_monkeyfunc, player_mocker_stopall player_data = [] - pi_listening_wav = os.path.join( - STATIC_SOUNDS_PATH, "pi-listening.wav" - ) + pi_listening_wav = os.path.join(STATIC_SOUNDS_PATH, "pi-listening.wav") # Run player wavefile = [pi_listening_wav] for path in wavefile: diff --git a/tests/integrationtests/test_integration_tasker.py b/tests/integrationtests/test_integration_tasker.py index 6b3a827a..bd88fa8e 100644 --- a/tests/integrationtests/test_integration_tasker.py +++ b/tests/integrationtests/test_integration_tasker.py @@ -39,6 +39,11 @@ from scarlett_os.tasker import on_signal_recieved +# from scarlett_os.internal.debugger import enable_remote_debugging + +# enable_remote_debugging() + + @pytest.mark.scarlettonly @pytest.mark.scarlettonlyintgr class TestScarlettTasker(IntegrationTestbaseMainloop): diff --git a/tests/unittests/common/configure/test_ruamel_config.py b/tests/unittests/common/configure/test_ruamel_config.py index a37357b9..f0d604b2 100644 --- a/tests/unittests/common/configure/test_ruamel_config.py +++ b/tests/unittests/common/configure/test_ruamel_config.py @@ -24,6 +24,7 @@ # from ruamel_config import logging from tests.conftest import dict_compare +from tests import COMMON_MOCKED_CONFIG # source: @@ -281,3 +282,79 @@ def test_config_file_not_found(self): temp_config = ruamel_config.load_config(config_file) shutil.rmtree(base) + + +@pytest.mark.ruamelconfigonly +@pytest.mark.scarlettonly +@pytest.mark.unittest +@pytest.mark.simpleconfigtest +@pytest.mark.scarlettonlyunittest +class TestConfigManager(object): + + """ + Tests for the scarlett_os.common.configure.ruamel_config.ConfigManager class. + """ + + def test_config_path_base(self): + config_manager = ruamel_config.ConfigManager("/tmp/config.yaml") + assert config_manager.config_path_base == "/tmp" + + base = tempfile.mkdtemp() + config_file = os.path.join(base, "config.yaml") + + config_manager.config_path_base = base + assert config_manager.config_path_base == base + + shutil.rmtree(base) + + def test_config_path_base_without_args_is_default_config_path(self): + config_manager = ruamel_config.ConfigManager() + assert config_manager.config_path_base == "/home/pi/.config/scarlett_os" + + def test_config_manager_env_override(self, monkeypatch): + base = tempfile.mkdtemp() + config_file = os.path.join(base, "config.yaml") + + with open(config_file, "wt") as f: + f.write(COMMON_MOCKED_CONFIG) + + monkeypatch.setenv("SCARLETT_OS_CONFIG_LATITUDE", "300") + monkeypatch.setenv("SCARLETT_OS_CONFIG_LONGITUDE", "200") + monkeypatch.setenv( + "SCARLETT_OS_CONFIG_POCKETSPHINX_HMM", "/tmp/model/en-us/en-us" + ) + monkeypatch.setenv("SCARLETT_OS_CONFIG_POCKETSPHINX_LM", "/tmp/lm/1473.lm") + monkeypatch.setenv("SCARLETT_OS_CONFIG_POCKETSPHINX_DICT", "/tmp/dict/1473.dic") + monkeypatch.setenv("SCARLETT_OS_CONFIG_DEVICE", "plughw:CARD=Device,DEV=0") + + config_manager = ruamel_config.ConfigManager(config_file) + config_manager.load() + + assert config_manager.cfg["latitude"] == "300" + assert config_manager.cfg["longitude"] == "200" + assert config_manager.cfg["pocketsphinx"]["hmm"] == "/tmp/model/en-us/en-us" + assert config_manager.cfg["pocketsphinx"]["lm"] == "/tmp/lm/1473.lm" + assert config_manager.cfg["pocketsphinx"]["dict"] == "/tmp/dict/1473.dic" + assert ( + config_manager.cfg["pocketsphinx"]["device"] == "plughw:CARD=Device,DEV=0" + ) + + shutil.rmtree(base) + + def test_config_manager_prep_default_config(self, monkeypatch): + base = tempfile.mkdtemp() + config_file = os.path.join(base, "config.yaml") + + monkeypatch.setattr(ruamel_config.ConfigManager, "CONFIG_PATH", config_file) + + + config_manager = ruamel_config.ConfigManager(config_file) + config_manager.prep_default_config() + + config_manager.load() + + # Don't need to check all the values, if this is true, the rest are as well. + assert config_manager.cfg["latitude"] == 40.7056308 + + + shutil.rmtree(base) diff --git a/tests/unittests/test_player.py b/tests/unittests/test_player.py index 8cd3b831..731b1921 100644 --- a/tests/unittests/test_player.py +++ b/tests/unittests/test_player.py @@ -198,8 +198,6 @@ def test_ScarlettPlayer_init_uri_is_valid_raises_exception_InvalidUri(self): def test_ScarlettPlayer_init_missing_gst_elements( self, mock_gst_elementfactory_make ): - path = os.path.join( - STATIC_SOUNDS_PATH, "pi-listening.wav" - ) + path = os.path.join(STATIC_SOUNDS_PATH, "pi-listening.wav") with pytest.raises(scarlett_os.exceptions.IncompleteGStreamerError): p = player.ScarlettPlayer(path, False, False) diff --git a/tests/unittests/test_tasker.py b/tests/unittests/test_tasker.py index a0327bad..c631ddb0 100644 --- a/tests/unittests/test_tasker.py +++ b/tests/unittests/test_tasker.py @@ -269,7 +269,9 @@ def connected_to_listener_cb(*args, **kwargs): @pytest.mark.unittest class TestSoundType(object): def test_soundtype_get_path(self): - path_to_sound = "/home/pi/dev/bossjones-github/scarlett_os/scarlett_os/data/sounds" + path_to_sound = ( + "/home/pi/dev/bossjones-github/scarlett_os/scarlett_os/data/sounds" + ) assert tasker.STATIC_SOUNDS_PATH == path_to_sound assert type(tasker.SoundType.get_path("pi-cancel")) == list assert tasker.SoundType.get_path("pi-cancel") == [