Facebook/Messenger images

CHANGES.md

@@ -1,8 +1,10 @@
 # Change Log
 
-## 2017-??-?? : v2.?
+## 2017-??-?? : v2.5
 
 * Made the kissing emoticons (e.g. :*) result in `kissing_heart` instead of `kiss` https://github.com/iamcal/emoji-data/pull/73
+* Added support for Facebook and FB Messenger images. Only messenger currently has any images https://github.com/iamcal/emoji-data/pull/76
+* ?
 
 ## 2016-11-11 : v2.4.5
 

README.md

@@ -3,14 +3,17 @@ images for use on the web.
 
 You can see a catalog of the emoji data here: http://unicodey.com/emoji-data/table.htm
 
+## Image Sources
+
 Images are extracted from their sources and this library attempts to track the latest
-available versions. If you're looking for older versions of Apple of Android images
+available versions. If you're looking for older versions of Apple or Android images
 (such as the Hairy Heart) then you'll need to look at previous revisions.
 
 * Apple Emoji: Copyright © Apple Inc. - OS X 10.11.1
-* Android Emoji: Copyright © [The Android Open Source Project](https://s3-eu-west-1.amazonaws.com/tw-font/android/NOTICE) - Lollipop
-* Twitter Emoji Copyright © Twitter, Inc. - The original release
+* Android Emoji: Copyright © [The Android Open Source Project](https://s3-eu-west-1.amazonaws.com/tw-font/android/NOTICE) - Nougat
+* Twitter Emoji: Copyright © Twitter, Inc. - The original release
 * Emoji One Emoji: Copyright © [Ranks.com Inc.](http://www.emojione.com/developers) - master as of 2015-03-05
+* Facebook/Messenger Emoji: Copyright © Facebook, Inc. - v7
 
 ## Libraries which use this data
 
@@ -54,6 +57,8 @@ look like this:
 			"has_img_google": true,
 			"has_img_twitter": true,
 			"has_img_emojione": false,
+			"has_img_facebook": false,
+			"has_img_messenger": false,
 			"skin_variations": {
 				"261D-1F3FB": {
 					"unified": "261D-1F3FB",
@@ -64,6 +69,8 @@ look like this:
 					"has_img_google": false,
 					"has_img_twitter": false,
 					"has_img_emojione": false
+					"has_img_facebook": false
+					"has_img_messenger": false
 				},
 				...
 			}

build/build_image.php

@@ -38,6 +38,8 @@
 	create_sheet('twitter');
 	create_sheet('google');
 	create_sheet('emojione');
+	create_sheet('facebook');
+	create_sheet('messenger');
 
 
 	function create_sheet($type){

build/build_map.php

@@ -385,6 +385,8 @@ function simple_row($img_key, $shorts, $props){
 		$ret['has_img_google']		= file_exists("{$GLOBALS['dir']}/../img-google-64/{$ret['image']}");
 		$ret['has_img_twitter']		= file_exists("{$GLOBALS['dir']}/../img-twitter-64/{$ret['image']}");
 		$ret['has_img_emojione']	= file_exists("{$GLOBALS['dir']}/../img-emojione-64/{$ret['image']}");
+		$ret['has_img_facebook']	= file_exists("{$GLOBALS['dir']}/../img-facebook-64/{$ret['image']}");
+		$ret['has_img_messenger']	= file_exists("{$GLOBALS['dir']}/../img-messenger-64/{$ret['image']}");
 
 		foreach ($props as $k => $v) $ret[$k] = $v;
 
@@ -406,6 +408,8 @@ function simple_row($img_key, $shorts, $props){
 					'has_img_google'	=> file_exists("{$GLOBALS['dir']}/../img-google-64/{$var_img}"),
 					'has_img_twitter'	=> file_exists("{$GLOBALS['dir']}/../img-twitter-64/{$var_img}"),
 					'has_img_emojione'	=> file_exists("{$GLOBALS['dir']}/../img-emojione-64/{$var_img}"),
+					'has_img_facebook'	=> file_exists("{$GLOBALS['dir']}/../img-facebook-64/{$var_img}"),
+					'has_img_messenger'	=> file_exists("{$GLOBALS['dir']}/../img-messenger-64/{$var_img}"),
 				);
 
 				$ret['skin_variations'][$var_uni] = $variation;

build/build_sheets.php

@@ -1,21 +1,14 @@
 <?php
 	$dir = dirname(__FILE__).'/..';
 
-	build_sheet('apple', 32);
-	build_sheet('apple', 20);
-	build_sheet('apple', 16);
+	$sets = array('apple', 'twitter', 'google', 'emojione', 'facebook', 'messenger');
+	$sizes = array(16, 20, 32);
 
-	build_sheet('twitter', 32);
-	build_sheet('twitter', 20);
-	build_sheet('twitter', 16);
-
-	build_sheet('google', 32);
-	build_sheet('google', 20);
-	build_sheet('google', 16);
-
-	build_sheet('emojione', 32);
-	build_sheet('emojione', 20);
-	build_sheet('emojione', 16);
+	foreach ($sets as $set) {
+		foreach ($sizes as $size) {
+			build_sheet($set, $size);
+		}
+	}
 
 	function build_sheet($type, $size){
 

build/build_table.php

@@ -2,7 +2,6 @@
 	$in = file_get_contents('../emoji.json');
 	$catalog = json_decode($in, true);
 
-
 	function format_codepoints($raw){
 
 		if (!$raw) return '-';
@@ -54,19 +53,18 @@ function emoji_utf8_bytes($cp){
 ?>
 <html>
 <head>
-<meta http-equiv="content-type" content="text/html; charset=UTF-8" />
+<meta charset="UTF-8" />
 <title>Emoji Catalog</title>
 <link rel="stylesheet" type="text/css" media="all" href="emoji.css" />
 <style type="text/css">
 
 body {
     font-size: 12px;
-    font-family: Arial, Helvetica, sans-serif;
+    font-family: Helvetica Neue, Helvetica, Arial, sans-serif;
 }
 
 table {
-    -webkit-border-radius: 0.41em;
-    -moz-border-radius: 0.41em;
+    border-radius: 0.41em;
     border: 1px solid #999;
     font-size: 12px;
 }
@@ -101,7 +99,7 @@ function emoji_utf8_bytes($cp){
 
 <table cellspacing="0" cellpadding="0">
 	<tr>
-		<th colspan="6">Name</th>
+		<th colspan="8">Name</th>
 		<th>Short Name</th>
 		<th>ASCII</th>
 		<th>Unified</th>
@@ -117,16 +115,20 @@ function emoji_utf8_bytes($cp){
 <?php
 	foreach ($catalog as $row){
 
-		$url_apple    = $row['has_img_apple'   ] ? "img-apple-64/{$row['image']}"    : '';
-		$url_google   = $row['has_img_google'  ] ? "img-google-64/{$row['image']}"   : '';
-		$url_twitter  = $row['has_img_twitter' ] ? "img-twitter-64/{$row['image']}"  : '';
-		$url_emojione = $row['has_img_emojione'] ? "img-emojione-64/{$row['image']}" : '';
+		$url_apple     = $row['has_img_apple'    ] ? "img-apple-64/{$row['image']}"     : '';
+		$url_google    = $row['has_img_google'   ] ? "img-google-64/{$row['image']}"    : '';
+		$url_twitter   = $row['has_img_twitter'  ] ? "img-twitter-64/{$row['image']}"   : '';
+		$url_emojione  = $row['has_img_emojione' ] ? "img-emojione-64/{$row['image']}"  : '';
+		$url_facebook  = $row['has_img_facebook' ] ? "img-facebook-64/{$row['image']}"  : '';
+		$url_messenger = $row['has_img_messenger'] ? "img-messenger-64/{$row['image']}" : '';
 
 		echo "\t<tr>\n";
 		echo "\t\t<td><img src=\"{$url_apple}\" width=\"20\" height=\"20\" /></td>\n";
 		echo "\t\t<td><img src=\"{$url_google}\" width=\"20\" height=\"20\" /></td>\n";
 		echo "\t\t<td><img src=\"{$url_twitter}\" width=\"20\" height=\"20\" /></td>\n";
 		echo "\t\t<td><img src=\"{$url_emojione}\" width=\"20\" height=\"20\" /></td>\n";
+		echo "\t\t<td><img src=\"{$url_facebook}\" width=\"20\" height=\"20\" /></td>\n";
+		echo "\t\t<td><img src=\"{$url_messenger}\" width=\"20\" height=\"20\" /></td>\n";
 		echo "\t\t<td>".unicode_bytes($row['unified'])."</td>\n";
 		echo "\t\t<td>".HtmlSpecialChars(StrToLower($row['name']))."</td>\n";
 

build/facebook/README.md

@@ -0,0 +1,19 @@
+# Building the Facebook emoji images
+
+From any machine:
+
+    php grab.php
+
+This will fetch the 128px versions of all available emoji.
+
+Next you'll want to cut the 64px versions that are used in the sheets:
+
+    php make64.php
+
+And finally optimize them. This requires the optimizing tools in `build/README.md` to be installed
+and takes a very long time:
+
+    parallel ../optimize.sh ::: ../../img-facebook-64/*
+    parallel ../optimize.sh ::: ../../img-messenger-64/*
+
+The resulting 64px images are then ready to use.

build/facebook/grab.php

@@ -0,0 +1,164 @@
+<?php
+
+$json = file_get_contents('../../emoji.json');
+$data = json_decode($json, true);
+
+# comment these out to do incremental download
+#shell_exec("rm -f ../../img-messenger-128/*.png");
+#shell_exec("rm -f ../../img-facebook-128/*.png");
+
+
+foreach ($data as $row){
+	if (strlen($row['image'])){
+		fetch($row['image']);
+	}
+
+	if (isset($row['skin_variations'])){
+		foreach ($row['skin_variations'] as $row2){
+			fetch($row2['image']);
+		}
+	}
+}
+
+function fetch($img){
+	fetch_single($img, 'MESSENGER', 'img-messenger-128', 128);
+	#fetch_single($img, 'FBEMOJI', 'img-facebook-72', 72);
+}
+
+
+function fetch_single($img, $type_key, $dir, $size){
+	/* based upon Twitter scraper and Javascript from Facebook see Github issue #56 */
+
+	# files get stored here
+	$path = "../../{$dir}/{$img}";
+
+	# use this for just fetching missing images
+	if (file_exists($path)) return;
+
+	# Emoji Config
+	$supportedSizes = [16, 18, 20, 24, 28, 30, 32, 64, 128];
+	$types = array('FBEMOJI' => 'f', 'FB_EMOJI_EXTENDED' => 'e', 'MESSENGER' => 'z', 'UNICODE' => 'u');
+
+	$type = $types[$type_key];
+
+
+	# emoji.img = acbd-1234.png
+	# facebook  = abcd_1234.png
+
+	$url = build_url($type, $size, str_replace('-', '_', $img));
+
+	if (try_fetch($url, $path)){
+		echo '.';
+		return;
+	}
+
+	#echo "X[{$img}]";
+	echo 'X';
+	#sleep(1);
+}
+
+
+function build_url($type, $size, $img){
+
+	$pixelRatio = 1;
+	$schemaAuth = "https://www.facebook.com/images/emoji.php/v7";
+
+	$path = $pixelRatio . '/' . $size . '/' . $img;
+	$check = checksum($path);
+	$url = $schemaAuth . '/' . $type . $check . '/' . $path;
+
+	return $url;
+}
+
+function try_fetch($url, $path){
+
+	http_fetch($url, $path);
+
+	if (!file_exists($path)){
+		return false;
+	}
+
+	if (!filesize($path)){
+		@unlink($path);
+		return false;
+	}
+
+	# PNG signature?
+	$fp = fopen($path, 'r');
+	$sig = fread($fp, 4);
+	fclose($fp);
+
+	if ($sig != "\x89PNG"){
+		#print_r($sig);
+		@unlink($path);
+		return false;
+	}
+
+	# ok!
+	return true;
+}
+function encodeURIComponent($str) { /* a standard method in Javascript */
+	return $str;
+}
+function unescape($str) {
+	$trans = array('&amp;' => '&', '&lt;' => '<', '&gt;' => '>', '&quot;' => '"', '&#x27;' => "'");
+	return strtr($str, $trans);
+}
+function checksum($subpath) {
+	$checksumBase = 317426846;
+	$base = $checksumBase;
+//	$subpath = unescape(encodeURIComponent($subpath)); 
+	for ($pos = 0; $pos < strlen($subpath); $pos++) {
+		$base = ($base << 5) - $base + ord(substr($subpath, $pos, 1));
+		$base &= 4294967295;
+	}
+	return base_convert(($base & 255), 10, 16);
+}
+
+
+function http_fetch($url, $filename){
+
+	$fh = fopen($filename, 'w');
+
+	$options = array(
+		CURLOPT_FILE	=> $fh,
+		CURLOPT_TIMEOUT	=> 60,
+		CURLOPT_URL	=> $url,
+	);
+
+	$options[CURLOPT_HTTPHEADER] = array(
+		'Referer: https://www.facebook.com/',
+		'User-Agent: Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36',
+	);
+
+	$ch = curl_init();
+	curl_setopt_array($ch, $options);
+	curl_exec($ch);
+	$ret = curl_getinfo($ch);
+	curl_close($ch);
+
+	fclose($fh);
+
+	# show http error code?
+	#echo "({$ret['http_code']})";
+
+	#print_r($ret);
+	#exit;
+}
+
+
+function http_fetch_old($url, $filename){
+
+	$out = array();
+	$ret = 0;
+	exec("wget -qO {$filename} \"{$url}\"", $out, $ret);
+
+	# error from wget?
+	if ($ret){
+		#print_r($out);
+		@unlink($path);
+		return false;
+	}
+
+	return true;
+}

build/facebook/make64.php

@@ -0,0 +1,21 @@
+<?php
+	thumb_set('img-messenger-128', 'img-messenger-64');
+
+
+	function thumb_set($from, $to){
+
+		$files = glob("../../{$from}/*.png");
+		shell_exec("rm -f ../../{$to}/*.png");
+		foreach ($files as $src){
+			$bits = explode('/', $src);
+			$dst = "../../{$to}/".array_pop($bits);
+			exec("convert {$src} -resize 64x64 png32:{$dst}", $out, $code);
+			if ($code){
+				echo "ERROR:\n";
+				echo "   ".$out."\n";
+			}else{
+				echo ".";
+			}
+		}
+		echo "All done\n";
+	}

build/find_flags.php

@@ -7,8 +7,10 @@
 	$b = find_flags("../img-google-64/1f1*-1f1*.png");
 	$c = find_flags("../img-twitter-64/1f1*-1f1*.png");
 	$d = find_flags("../img-emojione-64/1f1*-1f1*.png");
+	$e = find_flags("../img-facebook-64/1f1*-1f1*.png");
+	$f = find_flags("../img-messenger-64/1f1*-1f1*.png");
 
-	$all = array_unique(array_merge($a, $b, $c, $d));
+	$all = array_unique(array_merge($a, $b, $c, $d, $e, $f));
 	sort($all);
 
 	foreach ($all as $line) echo $line;

build/find_unused.php

@@ -41,6 +41,8 @@
 	scan_unused($files, '../img-google-64/');
 	scan_unused($files, '../img-twitter-64/');
 	scan_unused($files, '../img-emojione-64/');
+	scan_unused($files, '../img-facebook-64/');
+	scan_unused($files, '../img-messenger-64/');
 	echo "~FIN~\n";
 
 	function scan_unused($files, $path){