Final tidyups

Author: ColumPaget

Docs/Proxys.txt

@@ -8,9 +8,21 @@ If no authentication is needed to use the proxy then the '<username>:<password>@
 
 'type' can be:
 	http				A standard http proxy
-	https				An http proxy that uses encrypted connections
-	ssltunnel 	An ssh server that supports connection forwarding
+	https				An http proxy that uses encrypted connections (so is running on port 443 with native ssl)
+	httpc				An http proxy that the HTTP/1.1 'CONNECT' method
 	socks4      A socks4 proxy (openssh or tor using proxy)
 	socks5			A socks5 proxy
 
-All connections will then be forwarded through the proxy. 	
+All connections will then be forwarded through the proxy.
+
+This is particularly useful in combination with ssh. The ssh command:
+
+	ssh -D *:5050 user@myhost
+
+will connect to 'myhost' and open a socks4/socks5 proxy service locally on port 5050. Movgrab can then be told:
+
+	movgrab -p 127.0.0.1:5050 http://www.youtube.com/?watch=ejfelefffDD
+
+and all it's communications will be routed through ssh and appear to come from 'myhost'.
+
+

Docs/Usage.txt

@@ -36,14 +36,16 @@ This command works the same as the -s streaming command, except that the file wi
 
 -v increases the debugging level. Two -v will cause every webpage visited to be printed out for inspection. Three -v will cause some extra information about headers etc to be printed out.
 
--p allows you to specify a proxy server in http://username:password@url:port format. So, something like 'movgrab -p http://guest:password@myproxy.com:8080 . 'Username', 'password' and 'port' are optional if there is no user logon and the port is 80.
+-p allows you to specify a proxy server in http://username:password@url:port format. So, something like 'movgrab -p http://guest:password@myproxy.com:8080 . 'Username', 'password' and 'port' are optional if there is no user logon and the port is 80. Other proxy protocols are supported by replacing 'http:' with 'https', 'httpc', 'socks4' and 'socks5'. 'https' is the same as 'http' but expects the server to communicate using SSL/TLS from the get-go (port 443 style). 'httpc' uses the HTTP/1.1 'CONNECT' method.
+
 
 -x attempts to get around 'family filter' on some websites
 
 -P allows you to specify a 'Player program' (e.g. 'mplayer') to run when a certain percentage (default, 25%) of the file has been downloaded.
 
 -Pp allow you to set what percent of download to launch the 'Player program' at.
 
+-np <path> Will write a 'now playing' file for ICY web-radio services that announce the current artist/track.
 
 In some cases websites have more than one format of a video on offer. In general movgrab tries to extract Flash video (flv) first. If you want a different format, you can first see what the website offers by doing this:
 

libUseful-2.6/ConnectionChain.c

@@ -3,7 +3,7 @@
 #include "SpawnPrograms.h"
 #include "expect.h"
 
-const char *HopTypes[]={"none","direct","httptunnel","ssh","sshtunnel","socks4","socks5","shell","telnet",NULL};
+const char *HopTypes[]={"none","direct","httpc","ssh","sshtunnel","socks4","socks5","shell","telnet",NULL};
 typedef enum {CONNECT_HOP_NONE, CONNECT_HOP_TCP, CONNECT_HOP_HTTPTUNNEL, CONNECT_HOP_SSH, CONNECT_HOP_SSHTUNNEL, CONNECT_HOP_SOCKS4, CONNECT_HOP_SOCKS5, CONNECT_HOP_SHELL_CMD, CONNECT_HOP_TELNET} THopTypes;
 
 char *GlobalConnectionChain=NULL;

main.c

@@ -135,6 +135,7 @@ CheckSettings();
 if (StrValid(Proxy)) 
 {
 	if (strncmp(Proxy,"http:",5)==0) HTTPSetProxy(Proxy);
+	else if (strncmp(Proxy,"https:",6)==0) HTTPSetProxy(Proxy);
 	else if (! SetGlobalConnectionChain(Proxy))
 	{
 		printf("ERROR: Failed to set proxy settings to '%s'\n",Proxy);

servicetypes.c

@@ -1007,16 +1007,8 @@ break;
 case TYPE_DAILYMOTION:
 #define DAILYMOTION_ITEM "/mp4\",\"url\":\""
 #define DAILYMOTION_ITEM_END "\"}"
-#define DAILYMOTION_TITLE_START "<meta property=\"og:title\" content=\""
-#define DAILYMOTION_TITLE_END "\""
-
-
-  if (strstr(Tempstr,DAILYMOTION_TITLE_START))
-  {
-    GenericExtractFromLine(Tempstr, "Title",DAILYMOTION_TITLE_START,DAILYMOTION_TITLE_END,Vars, EXTRACT_DEQUOTE | EXTRACT_DEHTMLQUOTE);
-  }
-
 
+	GenericTitleExtract(Tempstr, Vars);
   if (strstr(Tempstr,DAILYMOTION_ITEM))
   {
 		Token=UnQuoteStr(Token,Tempstr);
@@ -1279,9 +1271,8 @@ case TYPE_NATGEO:
 #define NATGEO_ITEM_END "\'"
 #define NATGEO_FLV_START "class=\"ngs_video\" data-require=\"ngsPlayer\" data-options='{ \"slug\": \""
 #define NATGEO_FLV_END "\""
-#define NATGEO_TITLE_START "<meta property=\"og:title\" content=\""
-#define NATGEO_TITLE_END "\""
 
+	GenericTitleExtract(Tempstr, Vars);
 	if (strstr(Tempstr,NATGEO_ITEM_START))
 	{
 		GenericExtractFromLine(Tempstr, "tmp",NATGEO_ITEM_START,NATGEO_ITEM_END,Vars,EXTRACT_DESLASHQUOTE | EXTRACT_NOSPACES);
@@ -1294,30 +1285,17 @@ case TYPE_NATGEO:
 	{
 		GenericExtractFromLine(Tempstr, "item:flv",NATGEO_FLV_START,NATGEO_FLV_END,Vars,EXTRACT_NOSPACES);
 	}
-
-	if (strstr(Tempstr,NATGEO_TITLE_START))
-	{
-		GenericExtractFromLine(Tempstr, "Title",NATGEO_TITLE_START,NATGEO_TITLE_END,Vars,0);
-	}
-
 break;
 
 case TYPE_VIDEOBASH:
 #define VIDEOBASH_ITEMSTART "&amp;file=\" + 'http://' + '"
 #define VIDEOBASH_ITEMEND "\'"
-#define VIDEOBASH_TITLE_START "<meta property=\"og:title\" content=\""
-#define VIDEOBASH_TITLE_END "\""
 
+	GenericTitleExtract(Tempstr, Vars);
 	if (strstr(Tempstr,VIDEOBASH_ITEMSTART))
 	{
 		GenericExtractFromLine(Tempstr, "item:mp4",VIDEOBASH_ITEMSTART,VIDEOBASH_ITEMEND,Vars,EXTRACT_DEQUOTE | EXTRACT_NOSPACES);
 	}
-
-	if (strstr(Tempstr,VIDEOBASH_TITLE_START))
-	{
-		GenericExtractFromLine(Tempstr, "Title",VIDEOBASH_TITLE_START,VIDEOBASH_TITLE_END,Vars,0);
-	}
-
 break;
 
 

settings.c

@@ -31,8 +31,8 @@ fprintf(stdout,"'-U'		User-agent string\n");
 fprintf(stdout,"'-a'		Authentication info in Username:Password format.\n");
 fprintf(stdout,"'-q'		QUIET. No progress/informative output.\n");
 fprintf(stdout,"'-b'		Background. Fork into background and nohup\n");
-fprintf(stdout,"'-p'		address of http/https/socks4/socks5/sshtunnel proxy server in URL format.\n");
-fprintf(stdout,"'-proxy'		address of http/https/socks4/socks5/sstunnel proxy server in URL format.\n");
+fprintf(stdout,"'-p'		address of http/https/httpc/socks4/socks5 proxy server in URL format.\n");
+fprintf(stdout,"'-proxy'		address of http/https/httpc/socks4/socks5 proxy server in URL format.\n");
 fprintf(stdout,"'-w'		Wait for addresses to be entered on stdin.\n");
 fprintf(stdout,"'-st'		Connection inactivity timeout in seconds. Set high for sites that 'throttle'\n");
 fprintf(stdout,"'-tw <int>'		Set max width of item title in progress display (Default 50 chars)\n");
@@ -63,6 +63,8 @@ for (i=1; DownloadTypes[i] !=NULL; i++) fprintf(stdout,"%-20s %s\n",DownloadType
 
 fprintf(stdout,"\nThe -proxy argument has the form <protocol>:<username>:<password>@<host>:<port>. So, for example socks4:192.168.1.1:8080 or socks5:user1:secret@192.168.1.1:5050\n");
 
+fprintf(stdout,"\nAvailable proxy types are: 'http', 'https' (which is 'http' style proxy with port-443 style ssl), 'httpc' (which uses the http 1.1 'CONNECT' method), 'socks4' and 'socks5'. 'socks4' and 'socks5' can be used with the 'ssh -D' command to pipe all communications through an ssh tunnel.\n");
+
 fprintf(stdout,"\nIf a website is not in the list, try 'movgrab -t generic <url>'\n");
 fprintf(stdout,"\nMovgrab can also be used to stream from internet radio and pipe it into a player. E.g.'\n		movgrab -o - -q -proxy ssltunnel://guest:s3cr3t@sshserver:1022 http://schizoid.in/schizoid-psy.pls | mpg123 -\n");
 fprintf(stdout,"\nFeel free to email me and tell me if you've used this software!\n");