Support multiple registries and repositories

.gitignore

@@ -13,6 +13,7 @@ clib-build
 clib-update
 clib-upgrade
 clib-uninstall
+*.exe
 
 test/package/package-*
 !test/package/package-*.c

Building.md

@@ -0,0 +1,40 @@
+# Building clib from source
+
+## OSx
+
+```sh
+$ git clone https://github.com/clibs/clib.git /tmp/clib
+$ cd /tmp/clib
+$ make install
+```
+
+## Ubuntu or debian
+
+```sh
+# install libcurl
+$ sudo apt install libcurl4-gnutls-dev -qq
+# clone
+$ git clone https://github.com/clibs/clib.git /tmp/clib && cd /tmp/clib
+# build
+$ make
+# put on path
+$ sudo make install
+```
+
+## Windows (crosscompiling from linux)
+The docker image contains the mingw toolchain which is used to compile for windows.
+Curl needs to be built from source.
+```shell
+# Download and compile curl
+$ docker run --rm dockcross/windows-static-64-posix > dockcross-windows-x64
+$ cat dockcross-windows-x64
+$ chmod +x dockcross-windows-x64
+$ wget https://curl.haxx.se/download/curl-7.76.0.tar.gz
+$ tar xzf curl-*
+$ CURL_SRC=curl-*
+$ ./dockcross-windows-x64 bash -c 'cd '"$CURL_SRC"' && ./configure --prefix="/work/deps/curl" --host=x86_64-w64-mingw32.static --with-winssl --disable-dependency-tracking --disable-pthreads --enable-threaded-resolver --disable-imap --disable-pop3 --disable-smpt --disable-ldap --disable-mqtt --disable-smb'
+$ ./dockcross-windows-x64 bash -c 'cd '"$CURL_SRC"' && make'
+$ ./dockcross-windows-x64 bash -c 'cd '"$CURL_SRC"' && make install'
+$ git clone https://github.com/clibs/clib.git && cd clib
+$ ./dockcross-windows-x64 make all NO_PTHREADS=1 EXE=true
+```

Makefile

@@ -12,17 +12,16 @@ RM      = rm -f
 MKDIR   = mkdir -p
 
 SRC  = $(wildcard src/*.c)
-COMMON_SRC = $(wildcard src/common/*.c)
-ALL_SRC = $(wildcard src/*.c src/*.h src/common/*.c src/common/*.h test/package/*.c test/cache/*.c)
+COMMON_SRC = $(wildcard src/common/*.c src/registry/*.c src/repository/*.c)
+ALL_SRC = $(wildcard src/*.c src/*.h src/common/*.c src/common/*.h src/registry/*.c src/registry/*.h src/repository/*.h src/repository/*.c test/package/*.c test/cache/*.c)
 SDEPS = $(wildcard deps/*/*.c)
 ODEPS = $(SDEPS:.c=.o)
 DEPS = $(filter-out $(ODEPS), $(SDEPS))
 OBJS = $(DEPS:.c=.o)
-MAKEFILES = $(wildcard deps/*/Makefile)
 
 export CC
 
-CFLAGS += -std=c99 -Ideps -Wall -Wno-unused-function -U__STRICT_ANSI__
+CFLAGS += -std=c99 -Ideps -Isrc/common -Isrc/repository -Isrc/registry -Wall -Werror=return-type -Werror=implicit-function-declaration -Wno-unused-function -U__STRICT_ANSI__
 
 ifdef STATIC
 	CFLAGS  += -DCURL_STATICLIB  $(shell deps/curl/bin/curl-config --cflags)
@@ -48,7 +47,7 @@ all: $(BINS)
 
 build: $(BINS)
 
-$(BINS): $(SRC) $(MAKEFILES) $(OBJS)
+$(BINS): $(SRC) $(COMMON_SRC) $(OBJS)
 	$(CC) $(CFLAGS) -o $@ $(COMMON_SRC) src/$(@:.exe=).c $(OBJS) $(LDFLAGS)
 
 $(MAKEFILES):

Readme.md

@@ -7,58 +7,28 @@
 
   ![c package manager screenshot](https://i.cloudup.com/GwqOU2hh9Y.png)
 
-## Installation
-
-  Expects [libcurl](http://curl.haxx.se/libcurl/) to be installed and linkable.
-
-  With [homebrew](https://github.com/Homebrew/homebrew):
-
-```sh
-$ brew install clib
-```
-
-  Or [MacPorts](https://www.macports.org):
-
-```sh
-$ sudo port selfupdate
-$ sudo port install clib
-```
-
-  With git:
-
-```sh
-$ git clone https://github.com/clibs/clib.git /tmp/clib
-$ cd /tmp/clib
-$ make install
-```
+## About
 
-  Ubuntu:
+Basically the lazy-man's copy/paste promoting smaller C utilities, also
+serving as a nice way to discover these sort of libraries. From my experience
+C libraries are scattered all over the web and discovery is relatively poor. The footprint of these libraries is usually quite large and unfocused. The goal of `clibs` is to provide
+stand-alone "micro" C libraries for developers to quickly install without coupling
+to large frameworks.
 
-```sh
-# install libcurl
-$ sudo apt-get install libcurl4-gnutls-dev -qq
-# clone
-$ git clone https://github.com/clibs/clib.git /tmp/clib && cd /tmp/clib
-# build
-$ make
-# put on path
-$ sudo make install
-```
+You should use `clib(1)` to fetch these files for you and check them into your repository, the end-user and contributors should not require having `clib(1)` installed. This allows `clib(1)` to fit into any new or existing C workflow without friction.
 
-## About
+The [listing of packages](https://github.com/clibs/clib/wiki/Packages) acts as the "registry". The registry is used by `clib(1)` when searching for packages.
 
-  Basically the lazy-man's copy/paste promoting smaller C utilities, also
-  serving as a nice way to discover these sort of libraries. From my experience
-  C libraries are scattered all over the web and discovery is relatively poor. The footprint of these libraries is usually quite large and unfocused. The goal of `clibs` is to provide
-  stand-alone "micro" C libraries for developers to quickly install without coupling
-  to large frameworks.
+## Installation and building
+Binaries for `clib(1)` releases can be found at [releases](https://github.com/clibs/clib/releases/).
+For OSx and linux [libcurl](http://curl.haxx.se/libcurl/) should be installed and linkable.
+The windows binaries do not require any libraries to be installed.
 
-  You should use `clib(1)` to fetch these files for you and check them into your repository, the end-user and contributors should not require having `clib(1)` installed. This allows `clib(1)` to fit into any new or existing C workflow without friction.
+See [Building](Building.md) for instructions on how to build clib.
 
-  The wiki [listing of packages](https://github.com/clibs/clib/wiki/Packages) acts as the "registry" and populates the `clib-search(1)` results.
 
 ## Usage
-
+More detailed information on how to use `clib` can be found in [Usage](Usage.md).
 ```
   clib <command> [options]
 
@@ -78,13 +48,9 @@ $ sudo make install
     search [query]       Search for packages
     help <cmd>           Display help for cmd
 ```
+More information about the Command Line Interface can be found [here](https://github.com/clibs/clib/wiki/Command-Line-Interface).
 
-More about the Command Line Interface [here](https://github.com/clibs/clib/wiki/Command-Line-Interface).
-
-## Examples
-
- More examples and best practices at [BEST_PRACTICE.md](https://github.com/clibs/clib/blob/master/BEST_PRACTICE.md).
-
+### Example usage
  Install a few dependencies to `./deps`:
 
 ```sh
@@ -109,10 +75,26 @@ $ clib install ms file hash
 $ clib install visionmedia/mon visionmedia/every visionmedia/watch
 ```
 
-## clib.json
-
- Example of a clib.json explicitly listing the source:
+## Clib.json
+Information about a clib project or a package is stored in `clib.json`.
+In a project that uses `clib` to install dependencies a typical `clib.json` will only contain the required dependencies.
+It may look something like:
+```json
+{
+  "name": "Copy and Paste-Inator",
+  "version": "0.4.2",
+  "description": "Creates copies of yourself to do all of his waiting in lines for you.",
+  "dependencies": {
+    "clibs/buffer": "0.0.1",
+    "clibs/term": "0.0.1",
+    "jwerle/throw.h": "0.0.0"
+  }
+}
+```
 
+Packages that can be installed by `clib` should also provide a `clib.json`.
+It contains the files that should be installed by `clib` in `"src"`
+An example of a clib.json for a package may look like:
 ```json
 {
   "name": "term",
@@ -125,7 +107,7 @@ $ clib install visionmedia/mon visionmedia/every visionmedia/watch
 }
 ```
 
- Example of a clib.json for an executable:
+Example of a clib.json for an executable:
 
 ```json
 {

BEST_PRACTICE.md → Usage.md

@@ -3,13 +3,12 @@
 This page will cover:
 
  - [How to use libraries](#how-to-use-installed-libraries-for-your-project).
- - [Example Makefile](#example-makefile).
- - [Example `clib.json` for executables](#example-clibjson-for-executable-project).
+   - [Example Makefile](#example-makefile).
+   - [Example `clib.json` for executables](#example-packagejson-for-executable-project).
  - [Making your own library package](#making-your-own-libraries).
- - [Example `clib.json` for libraries](#example-clibjson-for-libraries).
+   - [Example `clib.json` for libraries](#example-packagejson-for-libraries).
  - [How to install/uninstall executables](#install-and-uninstall-executables-packages).
-
-For instructions on installation, check out the [README](https://github.com/clibs/clib#installation).
+- [How to fetch packages from other sources](#how-to-fetch-packages-from-other-sources).
 
 ## How to use installed libraries for your project
 
@@ -116,7 +115,7 @@ at a example `clib.json` file for your project: (executable package)
   "dependencies": {
     "stephenmathieson/trim.c": "0.0.2",
     "clibs/commander": "1.3.2",
-    "clibs/logger": "0.0.1",
+    "clibs/logger": "0.0.1"
   },
   "install": "make install",
   "uninstall": "make uninstall"
@@ -129,9 +128,9 @@ _**NOTE:** Make sure you have a release as the same version in your `clib.json`
 
 ## Making your own libraries
 
-Now that you know how to use libraries, heres how to make your own:
+Now that you know how to use libraries, here is how to make your own:
 
-Like before, heres a typical project directory tree:
+Like before, a typical project directory tree:
 
 ```
 your-library-c/
@@ -230,3 +229,41 @@ $ sudo clib-uninstall visionmedia/mon
 ```
 
 <br>
+
+## How to fetch packages from other sources
+By default `clib` uses the [listing of packages](https://github.com/clibs/clib/wiki/Packages) as the place to look for packages, the registry, and [github.com](https://github.com) for downloading packages.
+You can specify additional registries and download from other repositories than github.
+This might be useful when using `clib` to install a mix of private and public packages.
+
+### Adding additional registries
+Additional registries can be provided in the `clib.json` of a project.
+Currently github wiki's and gitlab &mdash; both [gitlab.com](https://www.gitlab.com) and self hosted &mdash; are supported.
+For gitlab the format is a bit complicated as it has to conform to the gitlab api.
+You should use the same format as the default registry but in a file in a repository instead of in a wiki.
+```json
+{
+   // other definitions
+   "registries": [
+      "https://gitlab.com/api/v4/projects/25447829/repository/files/README.md/raw?ref=master",
+      "https://github.com/awesome-org/clib/wiki/Packages"
+   ]
+}
+```
+
+_**CAUTION:** For gitlab, the url should be of the form `/project/<id>` and not `/group/repo` check [my-gitlab-registry](https://gitlab.com/nouwaarom/my-clib-registry) for an example._
+
+### Downloading from gitlab or private sources
+To download from some sources, authentication might be required.
+To facilitate this `clib_secrets.json` is used to store the credentials.
+
+```json
+{
+    "github.com": "GITHUB_API_TOKEN",
+    "github.example.com": "GITLAB_USER_TOKEN"
+}
+```
+
+Gitlab always requires a secret in order to use the API.
+The secret can be obtained by clicking your profile and then (Preferences -> Access Tokens) and create a token with only `read_repository` rights.
+
+_**TIP:** To prevent accidentally commiting your secrets add `clib_secrets.json` to `.gitignore` and use `clib_secrets.json.dist` to specify for which domains a secret is required._ 

clib.json

@@ -18,11 +18,10 @@
     "which": "0.1.3",
     "stephenmathieson/str-flatten.c": "0.0.4",
     "commander": "1.3.2",
-    "stephenmathieson/wiki-registry.c": "0.0.4",
     "stephenmathieson/case.c": "0.1.3",
     "jwerle/fs.c": "0.2.0",
     "stephenmathieson/str-replace.c": "0.0.6",
-    "strdup": "*",
+    "strdup": "0.1.*",
     "Constellation/console-colors.c": "1.0.1",
     "littlstar/asprintf.c": "0.0.3",
     "logger": "0.0.1",
@@ -37,9 +36,16 @@
     "stephenmathieson/debug.c": "0.0.0",
     "stephenmathieson/tempdir.c": "0.0.2",
     "isty001/copy": "0.0.0",
-    "stephenmathieson/rimraf.c": "0.1.0"
+    "stephenmathieson/rimraf.c": "0.1.0",
+    "thlorenz/gumbo-parser.c": "*",
+    "stephenmathieson/http-get.c": "*",
+    "stephenmathieson/gumbo-text-content.c": "*",
+    "stephenmathieson/gumbo-get-element-by-id.c": "*",
+    "stephenmathieson/gumbo-get-elements-by-tag-name.c": "*",
+    "clibs/list": "*",
+    "jwerle/url.h": "0.2.*"
   },
   "development": {
     "stephenmathieson/describe.h": "2.0.1"
   }
-}
+}

deps/http-get/http-get.c

@@ -40,7 +40,7 @@ static size_t http_get_cb(void *contents, size_t size, size_t nmemb, void *userp
   return realsize;
 }
 
-http_get_response_t *http_get_shared(const char *url, CURLSH *share) {
+http_get_response_t *http_get_shared(const char *url, CURLSH *share, const char** const headers, int header_count) {
   CURL *req = curl_easy_init();
 
   http_get_response_t *res = malloc(sizeof(http_get_response_t));
@@ -50,6 +50,15 @@ http_get_response_t *http_get_shared(const char *url, CURLSH *share) {
     curl_easy_setopt(req, CURLOPT_SHARE, share);
   }
 
+  if (header_count > 0) {
+    struct curl_slist *chunk = NULL;
+    for (int i = 0; i < header_count; i++) {
+      chunk = curl_slist_append(chunk, headers[i]);
+    }
+    /* set our custom set of headers */
+    curl_easy_setopt(req, CURLOPT_HTTPHEADER, chunk);
+  }
+
   curl_easy_setopt(req, CURLOPT_URL, url);
   curl_easy_setopt(req, CURLOPT_HTTPGET, 1);
   curl_easy_setopt(req, CURLOPT_FOLLOWLOCATION, 1);
@@ -70,8 +79,8 @@ http_get_response_t *http_get_shared(const char *url, CURLSH *share) {
  * Perform an HTTP(S) GET on `url`
  */
 
-http_get_response_t *http_get(const char *url) {
-  return http_get_shared(url, NULL);
+http_get_response_t *http_get(const char *url, const char** const headers, int header_count) {
+  return http_get_shared(url, NULL, headers, header_count);
 }
 
 /**
@@ -88,7 +97,7 @@ static size_t http_get_file_cb(void *ptr, size_t size, size_t nmemb, void *strea
  * Request `url` and save to `file`
  */
 
-int http_get_file_shared(const char *url, const char *file, CURLSH *share) {
+int http_get_file_shared(const char *url, const char *file, CURLSH *share, const char** const headers, int header_count) {
   CURL *req = curl_easy_init();
   if (!req) return -1;
 
@@ -99,6 +108,15 @@ int http_get_file_shared(const char *url, const char *file, CURLSH *share) {
     curl_easy_setopt(req, CURLOPT_SHARE, share);
   }
 
+  if (header_count > 0) {
+    struct curl_slist *chunk = NULL;
+    for (int i = 0; i < header_count; i++) {
+      chunk = curl_slist_append(chunk, headers[i]);
+    }
+    /* set our custom set of headers */
+    curl_easy_setopt(req, CURLOPT_HTTPHEADER, chunk);
+  }
+
   curl_easy_setopt(req, CURLOPT_URL, url);
   curl_easy_setopt(req, CURLOPT_HTTPGET, 1);
   curl_easy_setopt(req, CURLOPT_FOLLOWLOCATION, 1);
@@ -115,8 +133,8 @@ int http_get_file_shared(const char *url, const char *file, CURLSH *share) {
   return (200 == status && CURLE_ABORTED_BY_CALLBACK != res) ? 0 : -1;
 }
 
-int http_get_file(const char *url, const char *file) {
-  return http_get_file_shared(url, file, NULL);
+int http_get_file(const char *url, const char *file, const char** const headers, int header_count) {
+  return http_get_file_shared(url, file, NULL, NULL, 0);
 }
 
 /**