From 008b81920c526b82ab0351455b438e13faf6f3d5 Mon Sep 17 00:00:00 2001 From: Stefan Ernst Date: Fri, 11 Sep 2020 10:55:56 +0200 Subject: [PATCH 01/11] Progress bar starts correctly, fixed autogenerating max date --- .gitattributes | 0 .gitignore | 0 LICENSE | 0 bash/force-level1-csd.sh | 7 +++++-- 4 files changed, 5 insertions(+), 2 deletions(-) mode change 100644 => 100755 .gitattributes mode change 100644 => 100755 .gitignore mode change 100644 => 100755 LICENSE diff --git a/.gitattributes b/.gitattributes old mode 100644 new mode 100755 diff --git a/.gitignore b/.gitignore old mode 100644 new mode 100755 diff --git a/LICENSE b/LICENSE old mode 100644 new mode 100755 diff --git a/bash/force-level1-csd.sh b/bash/force-level1-csd.sh index 855d2847..d8f9b5ac 100755 --- a/bash/force-level1-csd.sh +++ b/bash/force-level1-csd.sh @@ -138,6 +138,7 @@ show_progress() { SIZEDONE=$(awk -v done=$SIZEDONE -v fsize=$FILESIZE 'BEGIN { print (done + fsize) }' ) PERCDONE=$(awk -v total=$TOTALSIZE -v done=$SIZEDONE 'BEGIN { printf( "%.2f\n", (100 / total * done) )}') local WIDTH=$(($(tput cols) - 9)) PERCINT=$(( $(echo $PERCDONE | cut -d"." -f1) + 1 )) + if [ $ITER -eq 1 ]; then local PERCINT=0; fi printf -v INCREMENT "%*s" "$(( $PERCINT*$WIDTH/100 ))" ""; INCREMENT=${INCREMENT// /=} printf "\r\e[K|%-*s| %3d %% %s" "$WIDTH" "$INCREMENT" "$PERCINT" "$*" } @@ -170,7 +171,7 @@ which_satellite() { # Initialize arguments and parse command line input SENSIN="LT04,LT05,LE07,LC08,S2A,S2B" DATEMIN="19700101" -DATEMAX=$(date +%F) +DATEMAX=$(date +%Y%m%d) CCMIN=0 CCMAX=100 TIER="T1" @@ -518,6 +519,8 @@ get_data() { fi FILESIZE=$(( $FILESIZEBYTE / 1048576 )) + show_progress + TILEPATH=$POOL/$TILE SCENEPATH=$TILEPATH/$SCENEID if [ $SATELLITE = "sentinel2" ]; then @@ -549,7 +552,7 @@ get_data() { echo "$SCENEPATH QUEUED" >> $QUEUE lockfile-remove $QUEUE - show_progress + #show_progress ((ITER++)) done fi From 639f8ddb26d04bfdd9e68421805a63fae4988f21 Mon Sep 17 00:00:00 2001 From: Stefan Ernst Date: Fri, 11 Sep 2020 10:58:39 +0200 Subject: [PATCH 02/11] removed uneccesary line --- bash/force-level1-csd.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bash/force-level1-csd.sh b/bash/force-level1-csd.sh index d8f9b5ac..0c567dcc 100755 --- a/bash/force-level1-csd.sh +++ b/bash/force-level1-csd.sh @@ -551,8 +551,7 @@ get_data() { lockfile-create $QUEUE echo "$SCENEPATH QUEUED" >> $QUEUE lockfile-remove $QUEUE - - #show_progress + ((ITER++)) done fi From fef92bfe03441698bf2b1092b1551655c0c2f5b0 Mon Sep 17 00:00:00 2001 From: David Frantz Date: Tue, 15 Sep 2020 09:16:44 +0200 Subject: [PATCH 03/11] updated Makefile for continued work on develop --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ec39e7bf..f3b8a7e8 100755 --- a/Makefile +++ b/Makefile @@ -24,7 +24,7 @@ # Modify the following lines to match your needs # Installation directory -BINDIR=/usr/local/bin +BINDIR=/develop # Libraries GDAL=-I/usr/include/gdal -L/usr/lib -Wl,-rpath=/usr/lib From 68ff66d66a906c6d1fa4580b10c005a4561f478a Mon Sep 17 00:00:00 2001 From: David Frantz Date: Tue, 15 Sep 2020 09:25:17 +0200 Subject: [PATCH 04/11] adding docs --- .../components/higher-level/tsa/param.rst | 3 + docs/source/history/v3.rst | 109 +++++++++++++++++- docs/source/history/vdev.rst | 102 +--------------- 3 files changed, 111 insertions(+), 103 deletions(-) diff --git a/docs/source/components/higher-level/tsa/param.rst b/docs/source/components/higher-level/tsa/param.rst index 0145248d..90689422 100755 --- a/docs/source/components/higher-level/tsa/param.rst +++ b/docs/source/components/higher-level/tsa/param.rst @@ -524,6 +524,9 @@ The following parameter descriptions are a print-out of ``force-parameter``, whi .. _tsa-param-polar: + +* **Land surface phenology parameters (Polar-based)** + .. note:: The Polar-based Land Surface Phenology (LSP) metrics are always available (they do not rely on SPLITS). diff --git a/docs/source/history/v3.rst b/docs/source/history/v3.rst index 9c6e0c95..33d163f2 100755 --- a/docs/source/history/v3.rst +++ b/docs/source/history/v3.rst @@ -1,7 +1,112 @@ .. _v3: -Version 3 -========= +FORCE v. 3.5.0 +-------------- + +Release: 15.09.2020 + + +* **General changes** + + * GDAL >= 3.0 support: + + Since GDAL >= 3.0, the coordinates from coordinate transformation operations are no longer sorted as X/Y or LON/LAT, but in the typical order of each coordinate system. + This has caused an incompatibility of FORCE with GDAL >= 3. + This is fixed now. Still, do not use FORCE < 3.5 with GDAL >= 3. Update FORCE instead. + + * Fixed a small bug that prevented program execution when the parameterfile was too long. + + * FORCE programs will now transition to a Unix-typical usage, where non-mandatory options are specified with -o or --long-option. + This will happen from time to time in the next releases. + The new program force-level1-csd is a prototype for this. + + * Reduced the amount of compiler warnings when compiling with gcc 9.3.0 under Ubuntu 20.40 LTS. + This is ongoing work. The goal is to get rid of all warnings (they are not critical, though). + +* **FORCE LEVEL 1 ARCHIVING SYSTEM** + + * new program force-level1-csd: + + The new force-csd tool (cloud storage download) was kindly contributed by Stefan Ernst. + This tool goes way beyond the capabilities of the former sensor-specific scripts force-level1-landsat and force-level1-sentinel2. + It downloads Sentinel-2 AND Landsat data from the Google Cloud Storage. + You need the gutils python package, and you might need an account on Google's end. + This is currently free. After communicating with Google, we expect this to remain free in the future. + force-csd is very efficient, you can use parallel downloads. + You can filter the datapool w.r.t. sensor, date, cloud cover, tier level etc. + The area of interest can be specified in a variety of ways, e.g. coordinate string as in force-level1-sentinel2, but a list of WRS-2/MGRS scenes, or vector geometries (e.g. shapefile) are also supported. + Of course, it also takes care about the file queues needed for Level 2 Processing. + + * deprecated programs: force-level1-landsat and force-level1-sentinel2: + + Due to the superiority of force-level1-csd, force-level1-landsat and force-level1-sentinel2 are now deprecated. + They will remain in the repo for a while, but will only receive minimal support in the future. + Users are encouraged to change their workflow accordingly. + +* **FORCE HIGHER LEVEL** + + * in force-higher-level, Level 3 sub-module: + + Included a safety query, which ensures that at least one score should be > 0. + Setting all scores to 0 resulted in some crashes. + Thanks to Jonas Ardö to report on this. + + * in force-higher-level, TSA sub-module: + + A new suite of metrics was implemented: Land Surface Phenology descriptors based on a polar transformation, Polarmetrics it is. + This is based on the paper by Bjorn-Gustaf J. Brooks: https://www.mdpi.com/1999-4907/11/6/606, but was modified (e.g. to consider interannual shifts in the start of the phenological year) and complemented by many more metrics. + A couple of new parameters are now necessary, force-parameter has been updated to include these in the skeleton. + See :ref:`tsa-param-polar`. + + * in force-higher-level, TSA sub-module: + + The domain tag of the bandwise FORCE metadata domain was updated to be interoperable with a 4D data model (see force-stack below). + + * in force-higher-level, TSA sub-module: + + When folding the time series, the quantile statistics were broken with a memory error. + This is fixed now. Thanks for Christoph Raab and Benjamin Jakimow for reporting this. + + * in force-higher-level, TSA sub-module, trend and CAT analyses: + + Added a new band to for computing relative change, i.e. gain/loss relative to initial value: (slope*timesteps)/offset. + This makes most sense when the index has a physical meaning like fractional cover. + This does not make sense at all when offset is negative! Take care. + + * in force-higher-level, TSA sub-module, trend and CAT analyses: + + fixed an issue that compromised signifance levels for trends, which were computed on DOY-based phenometrics. + +* **FORCE AUX** + + * in force-mosaic: + + force-mosaic now copies all metadata to the generated mosaics. + For this, the new aux tool force-mdcp is used + + * new program force-mdcp: + + This new helper tool copies metadata from one file to another. + This included the FORCE metadata domains. + This program is now used by force-mosaic to carry the metadata to the generated mosaics. + + * new program force-stack: + + This new tool stacks files in VRT format. + This works with physical images (e.g. GeoTiffs), as well as VRT files (as e.g. generated by force-mosaic). + A variable number of input files can be stacked, wildcards are supported. + Most basically, this is e.g. useful to look at RGB combinations in QGIS, where RGB visualizations cannot be used when the channels are not in the same file. + If the number of bands in the input files is different, the images are stacked after another, e.g. file 1 band 1, file 1 band 2, file 2 band 1. + If the number of bands in the input files is the same, the images are stacked with band interleave, e.g. file 1 band 1, file 2 band 1, file 1 band 2, file 2 band 2. + The second option implements a 4D data model (think of time series), which is consistent with the QGIS plugins Raster Time Series Manager and Raster Data Plotting (C) Andreas Rabe. + A tutorial is in development to illustrate the interoperable use between FORCE and said QGIS plugins. + + * in force-magic-parameters: + + The user can now change between "all combinations" or "paired combinations". + Please refer to the program description: + https://force-eo.readthedocs.io/en/latest/components/auxilliary/magic-parameters.htm + FORCE v. 3.4.0 -------------- diff --git a/docs/source/history/vdev.rst b/docs/source/history/vdev.rst index 95325bdf..caf8a186 100755 --- a/docs/source/history/vdev.rst +++ b/docs/source/history/vdev.rst @@ -8,104 +8,4 @@ FORCE-dev Master release: TBA -* **General changes** - - * GDAL >= 3.0 support: - - Since GDAL >= 3.0, the coordinates from coordinate transformation operations are no longer sorted as X/Y or LON/LAT, but in the typical order of each coordinate system. - This has caused an incompatibility of FORCE with GDAL >= 3. - This is fixed now. Still, do not use FORCE < 3.5 with GDAL >= 3. Update FORCE instead. - - * Fixed a small bug that prevented program execution when the parameterfile was too long. - - * FORCE programs will now transition to a Unix-typical usage, where non-mandatory options are specified with -o or --long-option. - This will happen from time to time in the next releases. - The new program force-level1-csd is a prototype for this. - - * Reduced the amount of compiler warnings when compiling with gcc 9.3.0 under Ubuntu 20.40 LTS. - This is ongoing work. The goal is to get rid of all warnings (they are not critical, though). - -* **FORCE LEVEL 1 ARCHIVING SYSTEM** - - * new program force-level1-csd: - - The new force-csd tool (cloud storage download) was kindly contributed by Stefan Ernst. - This tool goes way beyond the capabilities of the former sensor-specific scripts force-level1-landsat and force-level1-sentinel2. - It downloads Sentinel-2 AND Landsat data from the Google Cloud Storage. - You need the gutils python package, and you might need an account on Google's end. - This is currently free. After communicating with Google, we expect this to remain free in the future. - force-csd is very efficient, you can use parallel downloads. - You can filter the datapool w.r.t. sensor, date, cloud cover, tier level etc. - The area of interest can be specified in a variety of ways, e.g. coordinate string as in force-level1-sentinel2, but a list of WRS-2/MGRS scenes, or vector geometries (e.g. shapefile) are also supported. - Of course, it also takes care about the file queues needed for Level 2 Processing. - - * deprecated programs: force-level1-landsat and force-level1-sentinel2: - - Due to the superiority of force-level1-csd, force-level1-landsat and force-level1-sentinel2 are now deprecated. - They will remain in the repo for a while, but will only receive minimal support in the future. - Users are encouraged to change their workflow accordingly. - -* **FORCE HIGHER LEVEL** - - * in force-higher-level, Level 3 sub-module: - - Included a safety query, which ensures that at least one score should be > 0. - Setting all scores to 0 resulted in some crashes. - Thanks to Jonas Ardö to report on this. - - * in force-higher-level, TSA sub-module: - - A new suite of metrics was implemented: Land Surface Phenology descriptors based on a polar transformation, Polarmetrics it is. - This is based on the paper by Bjorn-Gustaf J. Brooks: https://www.mdpi.com/1999-4907/11/6/606, but was modified (e.g. to consider interannual shifts in the start of the phenological year) and complemented by many more metrics. - A couple of new parameters are now necessary, force-parameter has been updated to include these in the skeleton. - See :ref:`tsa-param-polar`. - - * in force-higher-level, TSA sub-module: - - The domain tag of the bandwise FORCE metadata domain was updated to be interoperable with a 4D data model (see force-stack below). - - * in force-higher-level, TSA sub-module: - - When folding the time series, the quantile statistics were broken with a memory error. - This is fixed now. Thanks for Christoph Raab and Benjamin Jakimow for reporting this. - - * in force-higher-level, TSA sub-module, trend and CAT analyses: - - Added a new band to for computing relative change, i.e. gain/loss relative to initial value: (slope*timesteps)/offset. - This makes most sense when the index has a physical meaning like fractional cover. - This does not make sense at all when offset is negative! Take care. - - * in force-higher-level, TSA sub-module, trend and CAT analyses: - - fixed an issue that compromised signifance levels for trends, which were computed on DOY-based phenometrics. - -* **FORCE AUX** - - * in force-mosaic: - - force-mosaic now copies all metadata to the generated mosaics. - For this, the new aux tool force-mdcp is used - - * new program force-mdcp: - - This new helper tool copies metadata from one file to another. - This included the FORCE metadata domains. - This program is now used by force-mosaic to carry the metadata to the generated mosaics. - - * new program force-stack: - - This new tool stacks files in VRT format. - This works with physical images (e.g. GeoTiffs), as well as VRT files (as e.g. generated by force-mosaic). - A variable number of input files can be stacked, wildcards are supported. - Most basically, this is e.g. useful to look at RGB combinations in QGIS, where RGB visualizations cannot be used when the channels are not in the same file. - If the number of bands in the input files is different, the images are stacked after another, e.g. file 1 band 1, file 1 band 2, file 2 band 1. - If the number of bands in the input files is the same, the images are stacked with band interleave, e.g. file 1 band 1, file 2 band 1, file 1 band 2, file 2 band 2. - The second option implements a 4D data model (think of time series), which is consistent with the QGIS plugins Raster Time Series Manager and Raster Data Plotting (C) Andreas Rabe. - A tutorial is in development to illustrate the interoperable use between FORCE and said QGIS plugins. - - * in force-magic-parameters: - - The user can now change between "all combinations" or "paired combinations". - Please refer to the program description: - https://force-eo.readthedocs.io/en/latest/components/auxilliary/magic-parameters.htm - \ No newline at end of file +No changes yet, master is in sync with develop. From f6505333d19fcb77ffc5b18b1f8f5fc1df107083 Mon Sep 17 00:00:00 2001 From: padanyi-gulyasg Date: Wed, 16 Sep 2020 11:10:49 +0200 Subject: [PATCH 05/11] Docker user fix + SPLITS default + smaller typo fix - Docker container runs as 'force-user' without root permission - Docker builds have SPLITS enabled by default from now on - Refreshed Docker documentation - Fixing small typo --- Dockerfile | 13 +++++++-- docs/source/setup/docker.rst | 44 ++++++++++++++++++++++++------ docs/source/setup/requirements.rst | 2 +- 3 files changed, 47 insertions(+), 12 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5aa8a436..93585abe 100755 --- a/Dockerfile +++ b/Dockerfile @@ -76,9 +76,9 @@ RUN ./configure CPPFLAGS="-I /usr/include/gdal" CXXFLAGS=-fpermissive \ RUN mkdir -p $INSTALL_DIR/force WORKDIR $INSTALL_DIR/force COPY . . -# Conditionally enable SPLITS which is disabled by default -ARG splits=false -RUN if [ "$splits" = "true" ] ; then ./splits.sh enable; else ./splits.sh disable; fi +# Conditionally disable SPLITS which is enabled by default +ARG splits=true +RUN if [ "$splits" = "false" ] ; then ./splits.sh disable; else ./splits.sh enable; fi # Conditionally enable DEBUG mode ARG debug=false RUN if [ "$debug" = "true" ] ; then ./debug.sh enable; else ./debug.sh disable; fi @@ -90,5 +90,12 @@ RUN make -j7 \ RUN rm -rf $INSTALL_DIR RUN apt-get purge -y --auto-remove apt-utils cmake git build-essential software-properties-common +# Create a dedicated user for running FORCE commands +RUN useradd -m force-user -p force && \ + chown -R force-user /usr/local/bin/ +# Use this user by default +USER force-user +WORKDIR /home/force-user + # Test FORCE run RUN force diff --git a/docs/source/setup/docker.rst b/docs/source/setup/docker.rst index 842957b6..c0f4e601 100644 --- a/docs/source/setup/docker.rst +++ b/docs/source/setup/docker.rst @@ -28,13 +28,13 @@ You can use FORCE like this: Which outputs a default message showing that FORCE works indeed. -On the `Docker hub page `_ you can see multiple tags for ``fegyi001/force``. The tags refer to the version first (e.g. v3.0.1), then optionally indicates whether it is a SPLITS version and/or a debug version. E.g. if you wish to use FORCE of version 3.0.1 in debug mode with SPLITS use this image: ``fegyi001/force:v3.0.1_splits_debug``. If you see only the version number this indicates default options (disabled SPLITS & disabled debug mode). +On the `Docker hub page `_ you can see multiple tags for ``fegyi001/force``. The tags refer to the version first (e.g. v3.0.1). E.g. if you wish to use FORCE of version 3.0.1 use this image: ``fegyi001/force:v3.0.1``. Local build ----------- -If you wish to build a Docker image instead of using the prebuilt version you can do it with the following steps from the root folder (this will not include SPLITS!): +If you wish to build a Docker image instead of using the prebuilt version you can do it with the following steps from the root folder: .. code-block:: bash @@ -42,11 +42,13 @@ If you wish to build a Docker image instead of using the prebuilt version you ca # The '-t' flag indicates how your local image will be called, in this case 'my-force' docker build -t my-force . -If you wish to enable **SPLITS** run the build with the following command: +There are optional build parameters for enabling/disabling SPLITS and/or DEBUG mode. By default SPLITS is enabled and DEBUG mode is disabled. + +If you wish to disable **SPLITS** run the build with the following command: .. code-block:: bash - docker build -t my-force --build-arg splits=true . + docker build -t my-force --build-arg splits=false . If you wish to enable **DEBUG** mode run the build with the following command: @@ -54,17 +56,24 @@ If you wish to enable **DEBUG** mode run the build with the following command: docker build -t my-force --build-arg debug=true . -You can add multiple build arguments, e.g. if you wish to enable SPLITS and DEBUG mode run the build with the following command: +You can add multiple build arguments, e.g. if you wish to disable SPLITS and enable DEBUG mode run the build with the following command: .. code-block:: bash - docker build -t my-force --build-arg splits=true --build-arg debug=true . + docker build -t my-force --build-arg splits=false --build-arg debug=true . + +Usage +----- -After that, you can use your newly built Docker image like this: +After downloading or building your own image, you can run it as a container like this: .. code-block:: bash + # using the prebuilt image + docker run fegyi001/force force + + # using a custom built image docker run my-force force The rest is up to you, you can do anything Docker containers support. E.g. you wish to add a volume to the container and run a ``force-level2`` command is as simple as that: @@ -74,5 +83,24 @@ The rest is up to you, you can do anything Docker containers support. E.g. you w # Let's say you have a parameter file in /my/local/folder/parameters.prm # You map your local folder into /opt/data for your force container # Without it FORCE will not be able to see your local files since it is isolated - docker run -v /my/local/folder:/opt/data my-force force-level2 /opt/data/parameters.prm + docker run -v /my/local/folder:/opt/data fegyi001/force force-level2 /opt/data/parameters.prm + +If you wish to enter the running container's terminal run it with the ``-it`` flag. In that case you can use this terminal just as you were on a Linux machine. + + .. code-block:: bash + + docker run -it -v /my/local/folder:/opt/data fegyi001/force + + +User credentials +---------------- + +If you have ``.scihub`` and ``.laads files`` on your local machine and you wish them to be used by FORCE in Docker you should attach the folder containing these files as a mounted volume, and set a Docker runtime environment variable pointing to that mounted folder location. This sounds complicated, but it really isn't: + + .. code-block:: bash + # --env sets the environment variable + # this command will only print the container's FORCE_CREDENTIALS variable + # is should result this: + # FORCE_CREDENTIALS=/app/credentials + docker run --env FORCE_CREDENTIALS=/app/credentials -v /path/to/credentials/folder/on/your/machine:/app/credentials fegyi001/force env | grep FORCE_CREDENTIALS \ No newline at end of file diff --git a/docs/source/setup/requirements.rst b/docs/source/setup/requirements.rst index bc66af56..75825386 100755 --- a/docs/source/setup/requirements.rst +++ b/docs/source/setup/requirements.rst @@ -22,7 +22,7 @@ Operating system The software was developed and tested under Ubuntu 12.04 LTS, 14.04 LTS, 16.04 LTS, and 18.04 LTS operating systems. Note that we do not provide support for migrating the code to any other OS. -The code can also be installed in other Linux dsitributions, e.g. CentOS. +The code can also be installed in other Linux distributions, e.g. CentOS. However, installation instructions are only given for Ubuntu. We were able to successfully install and run FORCE on the Windows 10 subsystem for Linux, too. From 0f77bbf500a9ce5d3aa89ddf8315e937b1e5f400 Mon Sep 17 00:00:00 2001 From: padanyi-gulyasg Date: Thu, 17 Sep 2020 09:54:21 +0200 Subject: [PATCH 06/11] Use docker user instead of force-docker --- Dockerfile | 32 ++++++++++++++++++-------------- docs/source/setup/docker.rst | 10 ++++++++++ 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/Dockerfile b/Dockerfile index 93585abe..c7ab335b 100755 --- a/Dockerfile +++ b/Dockerfile @@ -73,16 +73,19 @@ RUN ./configure CPPFLAGS="-I /usr/include/gdal" CXXFLAGS=-fpermissive \ && make clean # Build FORCE from source -RUN mkdir -p $INSTALL_DIR/force +RUN mkdir -p $INSTALL_DIR/force && \ + # This is needed in case of develop branch + mkdir -p /develop WORKDIR $INSTALL_DIR/force COPY . . -# Conditionally disable SPLITS which is enabled by default ARG splits=true -RUN if [ "$splits" = "false" ] ; then ./splits.sh disable; else ./splits.sh enable; fi -# Conditionally enable DEBUG mode ARG debug=false -RUN if [ "$debug" = "true" ] ; then ./debug.sh enable; else ./debug.sh disable; fi -RUN make -j7 \ +# Conditionally disable SPLITS which is enabled by default +RUN if [ "$splits" = "false" ] ; then ./splits.sh disable; else ./splits.sh enable; fi && \ + # Conditionally enable DEBUG mode + if [ "$debug" = "true" ] ; then ./debug.sh enable; else ./debug.sh disable; fi && \ + # Compile FORCE + make -j7 \ && make install \ && make clean @@ -90,12 +93,13 @@ RUN make -j7 \ RUN rm -rf $INSTALL_DIR RUN apt-get purge -y --auto-remove apt-utils cmake git build-essential software-properties-common -# Create a dedicated user for running FORCE commands -RUN useradd -m force-user -p force && \ - chown -R force-user /usr/local/bin/ -# Use this user by default -USER force-user -WORKDIR /home/force-user +# Create a dedicated 'docker' group and user for running FORCE commands +RUN groupadd docker && \ + useradd -m docker -g docker -p docker && \ + chgrp docker /usr/local/bin/ && \ + chgrp docker /develop + +WORKDIR /usr/local/bin -# Test FORCE run -RUN force +# In case of develop branch FORCE is installed into /develop folder +RUN if test -f "/develop/force"; then /develop/force; else force; fi \ No newline at end of file diff --git a/docs/source/setup/docker.rst b/docs/source/setup/docker.rst index c0f4e601..a6fbe1c1 100644 --- a/docs/source/setup/docker.rst +++ b/docs/source/setup/docker.rst @@ -90,6 +90,16 @@ If you wish to enter the running container's terminal run it with the ``-it`` fl .. code-block:: bash docker run -it -v /my/local/folder:/opt/data fegyi001/force + +It is highly recommended to use the Docker image with docker user. By default Docker uses the ``root`` user which can raise security issues. Therefore it is always advised to run FORCE in Docker with the ``-u`` (or ``--user``) flag like this: + + .. code-block:: bash + + # Uses the logged in user and group + docker run -it -u $(id -u):$(id -g) fegyi001/force bash + + # Uses a specific user and group + docker run -it -u 1000:1000 fegyi001/force bash User credentials From 430b7f32a4d0d664e813734760bdc99006a6f045 Mon Sep 17 00:00:00 2001 From: David Frantz Date: Thu, 17 Sep 2020 10:46:16 +0200 Subject: [PATCH 07/11] added loss = relative change to CAT parameters --- src/cross-level/enum-cl.c | 3 ++- src/cross-level/enum-cl.h | 3 ++- src/higher-level/trend-hl.c | 5 +++++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/cross-level/enum-cl.c b/src/cross-level/enum-cl.c index ea0d8bbe..05d23781 100755 --- a/src/cross-level/enum-cl.c +++ b/src/cross-level/enum-cl.c @@ -141,7 +141,8 @@ const tagged_enum_t _TAGGED_ENUM_TRD_[_TRD_LENGTH_] = { { _TRD_NUM_, "NUM"}}; const tagged_enum_t _TAGGED_ENUM_CAT_[_CAT_LENGTH_] = { - { _CAT_CHANGE_, "CHANGE"}, { _CAT_YEAR_, "YEAR-OF-CHANGE"}, + { _CAT_CHANGE_, "CHANGE"}, { _CAT_LOSS_, "LOSS"}, + { _CAT_YEAR_, "YEAR-OF-CHANGE"}, { _CAT_TOTAL_MEAN_, "TOTAL-MEAN"}, { _CAT_TOTAL_OFFSET_, "TOTAL-OFFSET"}, { _CAT_TOTAL_SLOPE_, "TOTAL-SLOPE"}, { _CAT_TOTAL_GAIN_, "TOTAL-GAIN"}, { _CAT_TOTAL_RSQ_, "TOTAL-RSQ"}, { _CAT_TOTAL_SIG_, "TOTAL-SIG"}, diff --git a/src/cross-level/enum-cl.h b/src/cross-level/enum-cl.h index 21fadf54..0b89d115 100755 --- a/src/cross-level/enum-cl.h +++ b/src/cross-level/enum-cl.h @@ -212,7 +212,8 @@ enum { _TRD_MEAN_, _TRD_OFFSET_, _TRD_SLOPE_, _TRD_NUM_, _TRD_LENGTH_ }; // change, aftereffect, trend -enum { _CAT_CHANGE_, _CAT_YEAR_, +enum { _CAT_CHANGE_, _CAT_LOSS_, + _CAT_YEAR_, _CAT_TOTAL_MEAN_, _CAT_TOTAL_OFFSET_, _CAT_TOTAL_SLOPE_, _CAT_TOTAL_GAIN_, _CAT_TOTAL_RSQ_, _CAT_TOTAL_SIG_, diff --git a/src/higher-level/trend-hl.c b/src/higher-level/trend-hl.c index 67000ea4..bd08d2b0 100755 --- a/src/higher-level/trend-hl.c +++ b/src/higher-level/trend-hl.c @@ -412,6 +412,11 @@ double mae, rmse; } + // loss, i.e. change relative to total offset + cat_[_CAT_LOSS_][p] = (short)(cat_[_CAT_CHANGE_][p]/ + cat_[_CAT_YEAR_+1+_TRD_LENGTH_*_PART_TOTAL_+_TRD_OFFSET_][p]* + 1000); + } } From 0c9fe11f3e872bedbe2846dc388214d36674b106 Mon Sep 17 00:00:00 2001 From: David Frantz Date: Thu, 17 Sep 2020 12:17:07 +0200 Subject: [PATCH 08/11] updated docs --- docs/source/setup/install.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/source/setup/install.rst b/docs/source/setup/install.rst index db85211a..4dbadf5e 100755 --- a/docs/source/setup/install.rst +++ b/docs/source/setup/install.rst @@ -66,6 +66,8 @@ Installation of the development version To install FORCE with all "bleeding-edge features", consider to use the develop version. * After point (3), change to the develop branch, then proceed with (4). + Note that `BINDIR` defaults to ``/develop`` on this branch. Not installing to ``/usr/local/bin`` (which is the default for the master branch) might make sense in the case you want to have both the master and develop versions installed. + You might want to change `BINDIR` to a directory that suits you (e.g. ``/usr/local/bin`` or a local directory). .. code-block:: bash From 66bbe570eeb3700383f287e1ade1da58d085d068 Mon Sep 17 00:00:00 2001 From: padanyi-gulyasg Date: Thu, 17 Sep 2020 14:22:51 +0200 Subject: [PATCH 09/11] forcing docker user, accessing executables --- Dockerfile | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index c7ab335b..09614355 100755 --- a/Dockerfile +++ b/Dockerfile @@ -85,6 +85,7 @@ RUN if [ "$splits" = "false" ] ; then ./splits.sh disable; else ./splits.sh enab # Conditionally enable DEBUG mode if [ "$debug" = "true" ] ; then ./debug.sh enable; else ./debug.sh disable; fi && \ # Compile FORCE + sed -i 's+BINDIR=/develop+BINDIR=/usr/local/bin+' Makefile && \ make -j7 \ && make install \ && make clean @@ -98,8 +99,9 @@ RUN groupadd docker && \ useradd -m docker -g docker -p docker && \ chgrp docker /usr/local/bin/ && \ chgrp docker /develop +# Use this user by default +USER docker -WORKDIR /usr/local/bin +WORKDIR /home/docker -# In case of develop branch FORCE is installed into /develop folder -RUN if test -f "/develop/force"; then /develop/force; else force; fi \ No newline at end of file +RUN force \ No newline at end of file From 1f35bd2a6e0b99999b5794b0f0b93f4353ac2acf Mon Sep 17 00:00:00 2001 From: padanyi-gulyasg Date: Thu, 17 Sep 2020 14:27:25 +0200 Subject: [PATCH 10/11] remove user-specific docker run --- docs/source/setup/docker.rst | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/docs/source/setup/docker.rst b/docs/source/setup/docker.rst index a6fbe1c1..c0f4e601 100644 --- a/docs/source/setup/docker.rst +++ b/docs/source/setup/docker.rst @@ -90,16 +90,6 @@ If you wish to enter the running container's terminal run it with the ``-it`` fl .. code-block:: bash docker run -it -v /my/local/folder:/opt/data fegyi001/force - -It is highly recommended to use the Docker image with docker user. By default Docker uses the ``root`` user which can raise security issues. Therefore it is always advised to run FORCE in Docker with the ``-u`` (or ``--user``) flag like this: - - .. code-block:: bash - - # Uses the logged in user and group - docker run -it -u $(id -u):$(id -g) fegyi001/force bash - - # Uses a specific user and group - docker run -it -u 1000:1000 fegyi001/force bash User credentials From 23b02839afeb09f5af97357c0cad5c3a6373e1f2 Mon Sep 17 00:00:00 2001 From: David Frantz Date: Thu, 17 Sep 2020 15:50:05 +0200 Subject: [PATCH 11/11] prepare 3.5.1 release --- Makefile | 2 +- README.md | 2 +- docs/source/index.rst | 2 +- src/cross-level/_version-cl.h | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index f3b8a7e8..ec39e7bf 100755 --- a/Makefile +++ b/Makefile @@ -24,7 +24,7 @@ # Modify the following lines to match your needs # Installation directory -BINDIR=/develop +BINDIR=/usr/local/bin # Libraries GDAL=-I/usr/include/gdal -L/usr/lib -Wl,-rpath=/usr/lib diff --git a/README.md b/README.md index 1c8cb3e1..a9f93118 100755 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ **Framework for Operational Radiometric Correction for Environmental monitoring** -**Version 3.5.0** +**Version 3.5.1** ![FORCE Logo](/images/force.png) diff --git a/docs/source/index.rst b/docs/source/index.rst index f8fa326c..df0ba34c 100755 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -5,7 +5,7 @@ FORCE documentation **FORCE: Framework for Operational Radiometric Correction for Environmental monitoring** -**Version 3.5.0** +**Version 3.5.1** `Download from Github `_. diff --git a/src/cross-level/_version-cl.h b/src/cross-level/_version-cl.h index 1acf0d64..bec16b5e 100755 --- a/src/cross-level/_version-cl.h +++ b/src/cross-level/_version-cl.h @@ -32,7 +32,7 @@ Version number extern "C" { #endif -#define _VERSION_ "3.5.0" +#define _VERSION_ "3.5.1" #ifdef __cplusplus }