MySQL Schema

urls

Column Name	Type	Remark
id	INT	Primary Key
hash	VARCHAR(32)	Unique, Index
url	TEXT
html	BLOB	Original HTML
text	BLOB	Extracted Text

SQL Export

CREATE TABLE `urls` (
  `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
  `hash` varchar(32) CHARACTER SET utf8 NOT NULL DEFAULT '',
  `url` text COLLATE utf8_unicode_ci NOT NULL,
  `html` longblob NOT NULL,
  `text` longblob NOT NULL,
  PRIMARY KEY (`id`),
  UNIQUE KEY `hash` (`hash`)
) ENGINE=InnoDB AUTO_INCREMENT=2067 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;