$banner

Directory listing for $dirname

- $html -

%s%s\r\n' % (name, suffix, name, suffix) - return string.Template(INDEX_TEMPLATE).substitute(dirname=dirname, html=html) - - def filter(self, handler): - path = urlparse.urlsplit(handler.path).path - if path.startswith('/'): - path = urllib.unquote_plus(path.lstrip('/') or '.').decode('utf8') - if os.path.isdir(path): - index_file = os.path.join(path, self.index_file) - if not os.path.isfile(index_file): - content = self.format_index_html(path).encode('UTF-8') - headers = {'Content-Type': 'text/html; charset=utf-8', 'Connection': 'close'} - return [handler.MOCK, 200, headers, content] - else: - path = index_file - if os.path.isfile(path): - content_type = 'application/octet-stream' - try: - import mimetypes - content_type = mimetypes.types_map.get(os.path.splitext(path)[1]) - except StandardError as e: - logging.error('import mimetypes failed: %r', e) - with open(path, 'rb') as fp: - content = fp.read() - headers = {'Connection': 'close', 'Content-Type': content_type} - return [handler.MOCK, 200, headers, content] - - -class BlackholeFilter(BaseProxyHandlerFilter): - """blackhole filter""" - one_pixel_gif = 'GIF89a\x01\x00\x01\x00\x80\xff\x00\xc0\xc0\xc0\x00\x00\x00!\xf9\x04\x01\x00\x00\x00\x00,\x00\x00\x00\x00\x01\x00\x01\x00\x00\x02\x02D\x01\x00;' - - def filter(self, handler): - if handler.command == 'CONNECT': - return [handler.STRIP, True, self] - elif handler.path.startswith(('http://', 'https://')): - headers = {'Cache-Control': 'max-age=86400', - 'Expires': 'Oct, 01 Aug 2100 00:00:00 GMT', - 'Connection': 'close'} - content = '' - if urlparse.urlsplit(handler.path).path.lower().endswith(('.jpg', '.gif', '.png','.jpeg', '.bmp')): - headers['Content-Type'] = 'image/gif' - content = self.one_pixel_gif - return [handler.MOCK, 200, headers, content] - else: - return [handler.MOCK, 404, {'Connection': 'close'}, ''] - - -class PACProxyHandler(SimpleProxyHandler): - """pac proxy handler""" - handler_filters = [PacFileFilter(), StaticFileFilter(), BlackholeFilter()] - - -def get_process_list(): - import ctypes - import collections - Process = collections.namedtuple('Process', 'pid name exe') - process_list = [] - if os.name == 'nt': - PROCESS_QUERY_INFORMATION = 0x0400 - PROCESS_VM_READ = 0x0010 - lpidProcess = (ctypes.c_ulong * 1024)() - cb = ctypes.sizeof(lpidProcess) - cbNeeded = ctypes.c_ulong() - ctypes.windll.psapi.EnumProcesses(ctypes.byref(lpidProcess), cb, ctypes.byref(cbNeeded)) - nReturned = cbNeeded.value/ctypes.sizeof(ctypes.c_ulong()) - pidProcess = [i for i in lpidProcess][:nReturned] - has_queryimage = hasattr(ctypes.windll.kernel32, 'QueryFullProcessImageNameA') - for pid in pidProcess: - hProcess = ctypes.windll.kernel32.OpenProcess(PROCESS_QUERY_INFORMATION | PROCESS_VM_READ, 0, pid) - if hProcess: - modname = ctypes.create_string_buffer(2048) - count = ctypes.c_ulong(ctypes.sizeof(modname)) - if has_queryimage: - ctypes.windll.kernel32.QueryFullProcessImageNameA(hProcess, 0, ctypes.byref(modname), ctypes.byref(count)) - else: - ctypes.windll.psapi.GetModuleFileNameExA(hProcess, 0, ctypes.byref(modname), ctypes.byref(count)) - exe = modname.value - name = os.path.basename(exe) - process_list.append(Process(pid=pid, name=name, exe=exe)) - ctypes.windll.kernel32.CloseHandle(hProcess) - elif sys.platform.startswith('linux'): - for filename in glob.glob('/proc/[0-9]*/cmdline'): - pid = int(filename.split('/')[2]) - exe_link = '/proc/%d/exe' % pid - if os.path.exists(exe_link): - exe = os.readlink(exe_link) - name = os.path.basename(exe) - process_list.append(Process(pid=pid, name=name, exe=exe)) - else: - try: - import psutil - process_list = psutil.get_process_list() - except StandardError as e: - logging.exception('psutil.get_process_list() failed: %r', e) - return process_list - -def pre_start(): - if not OpenSSL: - logging.warning('python-openssl not found, please install it!') - if sys.platform == 'cygwin': - logging.info('cygwin is not officially supported, please continue at your own risk :)') - #sys.exit(-1) - elif os.name == 'posix': - try: - import resource - resource.setrlimit(resource.RLIMIT_NOFILE, (8192, -1)) - except ValueError: - pass - elif os.name == 'nt': - import ctypes - ctypes.windll.kernel32.SetConsoleTitleW(u'GoAgent v%s' % __version__) - if not common.LISTEN_VISIBLE: - ctypes.windll.user32.ShowWindow(ctypes.windll.kernel32.GetConsoleWindow(), 0) - else: - ctypes.windll.user32.ShowWindow(ctypes.windll.kernel32.GetConsoleWindow(), 1) - if common.LOVE_ENABLE and random.randint(1, 100) <= 5: - title = ctypes.create_unicode_buffer(1024) - ctypes.windll.kernel32.GetConsoleTitleW(ctypes.byref(title), len(title)-1) - ctypes.windll.kernel32.SetConsoleTitleW('%s %s' % (title.value, random.choice(common.LOVE_TIP))) - blacklist = {'360safe': False, - 'QQProtect': False, } - softwares = [k for k, v in blacklist.items() if v] - if softwares: - tasklist = '\n'.join(x.name for x in get_process_list()).lower() - softwares = [x for x in softwares if x.lower() in tasklist] - if softwares: - title = u'GoAgent 建议' - error = u'某些安全软件(如 %s)可能和本软件存在冲突，造成 CPU 占用过高。\n如有此现象建议暂时退出此安全软件来继续运行GoAgent' % ','.join(softwares) - ctypes.windll.user32.MessageBoxW(None, error, title, 0) - #sys.exit(0) - if os.path.isfile('/proc/cpuinfo'): - with open('/proc/cpuinfo', 'rb') as fp: - m = re.search(r'(?im)(BogoMIPS|cpu MHz)\s+:\s+([\d\.]+)', fp.read()) - if m and float(m.group(2)) < 1000: - logging.warning("*NOTE*, Please set [gae]window=2 [gae]keepalive=1") - if GAEProxyHandler.max_window != common.GAE_WINDOW: - GAEProxyHandler.max_window = common.GAE_WINDOW - if common.GAE_KEEPALIVE and common.GAE_MODE == 'https': - GAEProxyHandler.ssl_connection_keepalive = True - if common.GAE_PAGESPEED and not common.GAE_OBFUSCATE: - logging.critical("*NOTE*, [gae]pagespeed=1 requires [gae]obfuscate=1") - sys.exit(-1) - if common.GAE_SSLVERSION: - GAEProxyHandler.ssl_version = getattr(ssl, 'PROTOCOL_%s' % common.GAE_SSLVERSION) - GAEProxyHandler.openssl_context = SSLConnection.context_builder(common.GAE_SSLVERSION) - if common.GAE_APPIDS[0] == 'goagent': - logging.critical('please edit %s to add your appid to [gae] !', common.CONFIG_FILENAME) - sys.exit(-1) - if common.GAE_MODE == 'http' and common.GAE_PASSWORD == '': - logging.critical('to enable http mode, you should set %r [gae]password = and [gae]options = rc4', common.CONFIG_FILENAME) - sys.exit(-1) - if common.GAE_TRANSPORT: - GAEProxyHandler.disable_transport_ssl = False - if common.GAE_REGIONS and not pygeoip: - logging.critical('to enable [gae]regions mode, you should install pygeoip') - sys.exit(-1) - if common.PAC_ENABLE: - pac_ip = ProxyUtil.get_listen_ip() if common.PAC_IP in ('', '::', '0.0.0.0') else common.PAC_IP - url = 'http://%s:%d/%s' % (pac_ip, common.PAC_PORT, common.PAC_FILE) - spawn_later(600, urllib2.build_opener(urllib2.ProxyHandler({})).open, url) - if not dnslib: - logging.error('dnslib not found, please put dnslib-0.8.3.egg to %r!', os.path.dirname(os.path.abspath(__file__))) - sys.exit(-1) - if not common.DNS_ENABLE: - if not common.HTTP_DNS: - common.HTTP_DNS = common.DNS_SERVERS[:] - for dnsservers_ref in (common.HTTP_DNS, common.DNS_SERVERS): - any(dnsservers_ref.insert(0, x) for x in [y for y in get_dnsserver_list() if y not in dnsservers_ref]) - AdvancedProxyHandler.dns_servers = common.HTTP_DNS - AdvancedProxyHandler.dns_blacklist = common.DNS_BLACKLIST - else: - AdvancedProxyHandler.dns_servers = common.HTTP_DNS or common.DNS_SERVERS - AdvancedProxyHandler.dns_blacklist = common.DNS_BLACKLIST - RangeFetch.threads = common.AUTORANGE_THREADS - RangeFetch.maxsize = common.AUTORANGE_MAXSIZE - RangeFetch.bufsize = common.AUTORANGE_BUFSIZE - RangeFetch.waitsize = common.AUTORANGE_WAITSIZE - if True: - GAEProxyHandler.handler_filters.insert(0, AutoRangeFilter()) - if common.GAE_REGIONS: - GAEProxyHandler.handler_filters.insert(0, DirectRegionFilter(common.GAE_REGIONS)) - if True: - GAEProxyHandler.handler_filters.insert(0, HostsFilter()) - if True: - GAEProxyHandler.handler_filters.insert(0, URLRewriteFilter()) - if common.HTTP_FAKEHTTPS: - GAEProxyHandler.handler_filters.insert(0, FakeHttpsFilter(common.HTTP_FAKEHTTPS, common.HTTP_NOFAKEHTTPS)) - PHPProxyHandler.handler_filters.insert(0, FakeHttpsFilter(common.HTTP_FAKEHTTPS, common.HTTP_NOFAKEHTTPS)) - if common.HTTP_FORCEHTTPS: - GAEProxyHandler.handler_filters.insert(0, ForceHttpsFilter(common.HTTP_FORCEHTTPS, common.HTTP_NOFORCEHTTPS)) - PHPProxyHandler.handler_filters.insert(0, ForceHttpsFilter(common.HTTP_FORCEHTTPS, common.HTTP_NOFORCEHTTPS)) - if common.HTTP_WITHGAE: - GAEProxyHandler.handler_filters.insert(0, WithGAEFilter(common.HTTP_WITHGAE)) - if common.USERAGENT_ENABLE: - GAEProxyHandler.handler_filters.insert(0, UserAgentFilter(common.USERAGENT_STRING)) - PHPProxyHandler.handler_filters.insert(0, UserAgentFilter(common.USERAGENT_STRING)) - if common.LISTEN_USERNAME: - GAEProxyHandler.handler_filters.insert(0, AuthFilter(common.LISTEN_USERNAME, common.LISTEN_PASSWORD)) - - -def main(): - global __file__ - __file__ = os.path.abspath(__file__) - if os.path.islink(__file__): - __file__ = getattr(os, 'readlink', lambda x: x)(__file__) - os.chdir(os.path.dirname(os.path.abspath(__file__))) - logging.basicConfig(level=logging.DEBUG if common.LISTEN_DEBUGINFO else logging.INFO, format='%(levelname)s - %(asctime)s %(message)s', datefmt='[%b %d %H:%M:%S]') - pre_start() - CertUtil.check_ca() - sys.stderr.write(common.info()) - - uvent_enabled = 'uvent.loop' in sys.modules and isinstance(gevent.get_hub().loop, __import__('uvent').loop.UVLoop) - - if common.PHP_ENABLE: - host, port = common.PHP_LISTEN.split(':') - HandlerClass = ((PHPProxyHandler, GreenForwardPHPProxyHandler) if not common.PROXY_ENABLE else (ProxyChainPHPProxyHandler, ProxyChainGreenForwardPHPProxyHandler))[uvent_enabled] - server = LocalProxyServer((host, int(port)), HandlerClass) - thread.start_new_thread(server.serve_forever, tuple()) - - if common.PAC_ENABLE: - server = LocalProxyServer((common.PAC_IP, common.PAC_PORT), PACProxyHandler) - thread.start_new_thread(server.serve_forever, tuple()) - - if common.DNS_ENABLE: - try: - sys.path += ['.'] - from dnsproxy import DNSServer - host, port = common.DNS_LISTEN.split(':') - server = DNSServer((host, int(port)), dns_servers=common.DNS_SERVERS, dns_blacklist=common.DNS_BLACKLIST, dns_tcpover=common.DNS_TCPOVER) - thread.start_new_thread(server.serve_forever, tuple()) - except ImportError: - logging.exception('GoAgent DNSServer requires dnslib and gevent 1.0') - sys.exit(-1) - - HandlerClass = ((GAEProxyHandler, GreenForwardGAEProxyHandler) if not common.PROXY_ENABLE else (ProxyChainGAEProxyHandler, ProxyChainGreenForwardGAEProxyHandler))[uvent_enabled] - server = LocalProxyServer((common.LISTEN_IP, common.LISTEN_PORT), HandlerClass) - try: - server.serve_forever() - except SystemError as e: - if '(libev) select: ' in repr(e): - logging.error('PLEASE START GOAGENT BY uvent.bat') - sys.exit(-1) - -if __name__ == '__main__': - main() diff --git a/proxy/goagent/local/proxy.sh b/proxy/goagent/local/proxy.sh deleted file mode 100755 index 3e3db5e..0000000 --- a/proxy/goagent/local/proxy.sh +++ /dev/null @@ -1,75 +0,0 @@ -#!/bin/sh -# -# goagent init script -# - -### BEGIN INIT INFO -# Provides: goagent -# Required-Start: $syslog -# Required-Stop: $syslog -# Should-Start: $local_fs -# Should-Stop: $local_fs -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: Monitor for goagent activity -# Description: goagent is a gae proxy forked from gappproxy/wallproxy. -### END INIT INFO - -# **NOTE** bash will exit immediately if any command exits with non-zero. -set -e - -PACKAGE_NAME=goagent -PACKAGE_DESC="goagent proxy server" -PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:${PATH} - -start() { - echo -n "Starting ${PACKAGE_DESC}: " - nohup /usr/bin/env python2.7 proxy.py 2>&1 | /usr/bin/logger -t ${PACKAGE_NAME} & - echo "${PACKAGE_NAME}." -} - -stop() { - echo -n "Stopping ${PACKAGE_DESC}: " - kill -9 `ps aux | grep 'python2.7 proxy.py' | grep -v grep | awk '{print $2}'` || true - echo "${PACKAGE_NAME}." -} - -restart() { - stop - sleep 1 - start -} - -usage() { - N=$(basename "$0") - echo "Usage: [sudo] $N {start|stop|restart}" >&2 - exit 1 -} - -if [ "$(id -u)" != "0" ]; then - echo "please use sudo to run ${PACKAGE_NAME}" - exit 0 -fi - -# `readlink -f` won't work on Mac, this hack should work on all systems. -cd $(python -c "import os; print os.path.dirname(os.path.realpath('$0'))") - -case "$1" in - # If no arg is given, start the goagent. - # If arg `start` is given, also start goagent. - '' | start) - start - ;; - stop) - stop - ;; - #reload) - restart | force-reload) - restart - ;; - *) - usage - ;; -esac - -exit 0 diff --git a/proxy/goagent/local/pygeoip-0.3.1.egg b/proxy/goagent/local/pygeoip-0.3.1.egg deleted file mode 100644 index 7932a1d..0000000 Binary files a/proxy/goagent/local/pygeoip-0.3.1.egg and /dev/null differ diff --git a/proxy/goagent/local/python27.dll b/proxy/goagent/local/python27.dll deleted file mode 100644 index 5d55cc5..0000000 Binary files a/proxy/goagent/local/python27.dll and /dev/null differ diff --git a/proxy/goagent/local/python27.exe b/proxy/goagent/local/python27.exe deleted file mode 100644 index e1c9119..0000000 Binary files a/proxy/goagent/local/python27.exe and /dev/null differ diff --git a/proxy/goagent/local/python27.zip b/proxy/goagent/local/python27.zip deleted file mode 100644 index 7000886..0000000 Binary files a/proxy/goagent/local/python27.zip and /dev/null differ diff --git a/src/.ycm_extra_conf.py b/src/.ycm_extra_conf.py deleted file mode 100644 index 313214b..0000000 --- a/src/.ycm_extra_conf.py +++ /dev/null @@ -1,152 +0,0 @@ -# This file is NOT licensed under the GPLv3, which is the license for the rest -# of YouCompleteMe. -# -# Here's the license text for this file: -# -# This is free and unencumbered software released into the public domain. -# -# Anyone is free to copy, modify, publish, use, compile, sell, or -# distribute this software, either in source code form or as a compiled -# binary, for any purpose, commercial or non-commercial, and by any -# means. -# -# In jurisdictions that recognize copyright laws, the author or authors -# of this software dedicate any and all copyright interest in the -# software to the public domain. We make this dedication for the benefit -# of the public at large and to the detriment of our heirs and -# successors. We intend this dedication to be an overt act of -# relinquishment in perpetuity of all present and future rights to this -# software under copyright law. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. -# -# For more information, please refer to - -import os -import ycm_core - -# These are the compilation flags that will be used in case there's no -# compilation database set (by default, one is not set). -# CHANGE THIS LIST OF FLAGS. YES, THIS IS THE DROID YOU HAVE BEEN LOOKING FOR. -flags = [ - '-std=c++11', - '-stdlib=libc++', - '-Werror', - '-Weverything', - '-Wno-documentation', - '-Wno-deprecated-declarations', - '-Wno-disabled-macro-expansion', - '-Wno-float-equal', - '-Wno-c++98-compat', - '-Wno-c++98-compat-pedantic', - '-Wno-global-constructors', - '-Wno-exit-time-destructors', - '-Wno-missing-prototypes', - '-Wno-padded', - '-x', - 'c++', - '-I', - '.', - '-I', - '/usr/include/', - '-I', - '/usr/include/c++/v1/' -] - - -# Set this to the absolute path to the folder (NOT the file!) containing the -# compile_commands.json file to use that instead of 'flags'. See here for -# more details: http://clang.llvm.org/docs/JSONCompilationDatabase.html -# -# Most projects will NOT need to set this to anything; you can just change the -# 'flags' list of compilation flags. Notice that YCM itself uses that approach. -compilation_database_folder = '' - -if compilation_database_folder: - database = ycm_core.CompilationDatabase( compilation_database_folder ) -else: - database = None - -SOURCE_EXTENSIONS = [ '.cpp', '.cxx', '.cc', '.c', '.m', '.mm' ] - -def DirectoryOfThisScript(): - return os.path.dirname( os.path.abspath( __file__ ) ) - - -def MakeRelativePathsInFlagsAbsolute( flags, working_directory ): - if not working_directory: - return list( flags ) - new_flags = [] - make_next_absolute = False - path_flags = [ '-isystem', '-I', '-iquote', '--sysroot=' ] - for flag in flags: - new_flag = flag - - if make_next_absolute: - make_next_absolute = False - if not flag.startswith( '/' ): - new_flag = os.path.join( working_directory, flag ) - - for path_flag in path_flags: - if flag == path_flag: - make_next_absolute = True - break - - if flag.startswith( path_flag ): - path = flag[ len( path_flag ): ] - new_flag = path_flag + os.path.join( working_directory, path ) - break - - if new_flag: - new_flags.append( new_flag ) - return new_flags - - -def IsHeaderFile( filename ): - extension = os.path.splitext( filename )[ 1 ] - return extension in [ '.h', '.hxx', '.hpp', '.hh' ] - - -def GetCompilationInfoForFile( filename ): - # The compilation_commands.json file generated by CMake does not have entries - # for header files. So we do our best by asking the db for flags for a - # corresponding source file, if any. If one exists, the flags for that file - # should be good enough. - if IsHeaderFile( filename ): - basename = os.path.splitext( filename )[ 0 ] - for extension in SOURCE_EXTENSIONS: - replacement_file = basename + extension - if os.path.exists( replacement_file ): - compilation_info = database.GetCompilationInfoForFile( - replacement_file ) - if compilation_info.compiler_flags_: - return compilation_info - return None - return database.GetCompilationInfoForFile( filename ) - - -def FlagsForFile( filename, **kwargs ): - if database: - # Bear in mind that compilation_info.compiler_flags_ does NOT return a - # python list, but a "list-like" StringVec object - compilation_info = GetCompilationInfoForFile( filename ) - if not compilation_info: - return None - - final_flags = MakeRelativePathsInFlagsAbsolute( - compilation_info.compiler_flags_, - compilation_info.compiler_working_dir_ ) - else: - relative_to = DirectoryOfThisScript() - final_flags = MakeRelativePathsInFlagsAbsolute( flags, relative_to ) - - return { - 'flags': final_flags, - 'do_cache': True - } diff --git a/src/.ycm_extra_conf.pyc b/src/.ycm_extra_conf.pyc deleted file mode 100644 index f8fd16e..0000000 Binary files a/src/.ycm_extra_conf.pyc and /dev/null differ diff --git a/src/lib/helper/CmdlineOption.cpp b/src/lib/helper/CmdlineOption.cpp deleted file mode 100644 index 765635f..0000000 --- a/src/lib/helper/CmdlineOption.cpp +++ /dev/null @@ -1,73 +0,0 @@ -// last modified - -#include "CmdlineOption.h" -#include -#include -#include -#include -#include - -using namespace std; - -static bool -isOption (const string& str) -{ - return( str.size() >= 3 && // the shortest option "--x" - '-' == str[0] && - '-' == str[1] && - '-' != str[2] ); -} - -// cmdname --foo aa, for example, --foo is an option, aa is an argument. -// convention about command line option: -// 0) option must begin with -- (so, the shortest option --x has three characters), and the argument cannot begin with --; -// 1) an argument must follow after an option; -// 2) an option can follow as an argument or not, E.G., some option for true or false. -// one option by one more arguments? E.G., --bar a b c d. It's ok. -CmdlineOption::CmdlineOption (unsigned argc, char* argv[]) -{ - if (argc < 2) { - return; - } - - vector raw_options_list(argv + 1, argv + argc); - - string last_option; - for (const auto& e : raw_options_list) { - if (isOption(e)) { - options_and_arguments_list_[e]; - last_option = e; - } else { - if (!last_option.empty()) { - options_and_arguments_list_[last_option].push_back(e); - } - } - } - -//// DEBUG. show the result of parsing command options -//for (const auto& e : options_and_arguments_list_) { - //const vector& arguments_list = e.second; - //cout << e.first << "(" << arguments_list.size() << "): "; - //copy(e.second.cbegin(), e.second.cend(), ostream_iterator(cout, ",")); - //cout << endl; -//} -} - -CmdlineOption::~CmdlineOption () -{ - ; -} - -bool -CmdlineOption::hasOption (const string& option) const -{ - return(options_and_arguments_list_.cend() != options_and_arguments_list_.find(option)); -} - -const vector& -CmdlineOption::getArgumentsList (const string& option) -{ - static const vector empty_arguments_list; - return(hasOption(option) ? options_and_arguments_list_[option] : empty_arguments_list); -} - diff --git a/src/lib/helper/CmdlineOption.h b/src/lib/helper/CmdlineOption.h deleted file mode 100644 index 7b05349..0000000 --- a/src/lib/helper/CmdlineOption.h +++ /dev/null @@ -1,24 +0,0 @@ -// last modified - -#pragma once -#include -#include -#include - -using std::string; -using std::pair; -using std::vector; -using std::unordered_map; - -class CmdlineOption -{ - public: - CmdlineOption (unsigned argc, char* argv[]); - virtual ~CmdlineOption (); - bool hasOption (const string& option) const; - const vector& getArgumentsList (const string& option); - - private: - unordered_map> options_and_arguments_list_; -}; - diff --git a/src/lib/helper/Misc.cpp b/src/lib/helper/Misc.cpp deleted file mode 100644 index 62350a7..0000000 --- a/src/lib/helper/Misc.cpp +++ /dev/null @@ -1,38 +0,0 @@ -// last modified - -#include "Misc.h" -#include - - -// split raw string to more sub-str by token-chars. -// note: -// 0) case sensitive; -// 1) if there are consecutive two token-chars in raw string, splitStr() -// will make a empty sub-str into splited_substr_list. -void -splitStr ( const string& str, - const string& tokens_list, - vector& splited_substr_list, - vector& appeared_tokens_list ) -{ - size_t begin_pos = 0, end_pos; - while (begin_pos < str.size()) { - const auto iter_token = find_first_of( str.cbegin() + (int)begin_pos, str.cend(), - tokens_list.cbegin(), tokens_list.cend() ); - if (str.cend() == iter_token) { - splited_substr_list.push_back(str.substr(begin_pos)); - break; - } - - appeared_tokens_list.push_back(*iter_token); - end_pos = (unsigned)(iter_token - str.cbegin()); - splited_substr_list.push_back(str.substr(begin_pos, end_pos - begin_pos)); - - begin_pos = end_pos + 1; - } - - if (splited_substr_list[0].empty()) { - splited_substr_list.erase(splited_substr_list.begin()); - } -} - diff --git a/src/lib/helper/Misc.h b/src/lib/helper/Misc.h deleted file mode 100644 index dee5ff5..0000000 --- a/src/lib/helper/Misc.h +++ /dev/null @@ -1,34 +0,0 @@ -// last modified - -#pragma once - -#include -#include -#include - -using std::string; -using std::ostringstream; -using std::vector; - - -// why not std::to_string()? -// you know, I have to port this linux code to win32 by cygwin, and there -// is a bug on cygwin case it cannot find to_string(), so, I must do it -// by myself -template -string -convNumToStr (T num) -{ - ostringstream oss; - oss << num; - - return(oss.str()); -} - -// split raw string to more sub-str by token-chars. -void -splitStr ( const string& str, - const string& tokens_list, - vector& splited_substr_list, - vector& appeared_tokens_list ); - diff --git a/src/lib/helper/RichTxt.h b/src/lib/helper/RichTxt.h deleted file mode 100644 index 4a04e97..0000000 --- a/src/lib/helper/RichTxt.h +++ /dev/null @@ -1,57 +0,0 @@ -// last modified - -#pragma once -#include - -using std::string; - -namespace RichTxt -{ - // bold - static const string bold_on("\x1b[1m"); - static const string bold_off("\x1b[21m"); - - // italic - static const string italic_on("\x1b[3m"); - static const string italic_off("\x1b[23m"); - - // underline - static const string underline_on("\x1b[4m"); - static const string underline_off("\x1b[24m"); - - // hide - static const string hide_on("\x1b[8m"); - static const string hide_off("\x1b[28m"); - - // deletline - static const string deletline_on("\x1b[9m"); - static const string deletline_off("\x1b[29m"); - - // foreground - static const string foreground_black("\x1b[30m"); - static const string foreground_red("\x1b[31m"); - static const string foreground_green("\x1b[32m"); - static const string foreground_yellow("\x1b[33m"); - static const string foreground_blue("\x1b[34m"); - static const string foreground_magenta("\x1b[35m"); - static const string foreground_cyan("\x1b[36m"); - static const string foreground_white("\x1b[37m"); - - // background - static const string background_black("\x1b[40m"); - static const string background_red("\x1b[41m"); - static const string background_green("\x1b[42m"); - static const string background_yellow("\x1b[43m"); - static const string background_blue("\x1b[44m"); - static const string background_magenta("\x1b[45m"); - static const string background_cyan("\x1b[46m"); - static const string background_white("\x1b[47m"); - - // reset all - static const string reset_all("\x1b[0m"); -}; - -// normal usage: -// 0) cout << "email: " << RichTxt::bold_on << "yangyang.gnu@gmail.com" << RichTxt::bold_off << endl; -// 1) string name("yangyang.gnu"); string name_italic = RichTxt::italic_on + RichTxt::background_green + name + RichTxt::italic_off; - diff --git a/src/lib/helper/Time.cpp b/src/lib/helper/Time.cpp deleted file mode 100644 index d612614..0000000 --- a/src/lib/helper/Time.cpp +++ /dev/null @@ -1,174 +0,0 @@ -// last modified - -#include "Time.h" -#include -#include -#include -#include - -using namespace std; - - -// why not std::to_string()? -// you know, I have to port this linux code to win32 by cygwin, and there -// is a bug on cygwin case it cannot find to_string(), so, I must do it by myself -static string -convUnsignedToStr (unsigned num) -{ - ostringstream oss; - oss << num; - - return(oss.str()); -} - -// string::resize() resize string from the first char to the last char, -// resizeStringByEndian() resize string from the last char to the first char -static string -resizeStringByEndian (const string& str, unsigned digits, char ch = '0') -{ - string strtmp(str.crbegin(), str.crend()); - strtmp.resize(digits, ch); - reverse(strtmp.begin(), strtmp.end()); - - return(strtmp); -} - -Time::Time () -{ - time_t raw_time = time(nullptr); - const struct tm* p_st = localtime(&raw_time); - - year_ = (unsigned)p_st->tm_year + 1900; - month_ = (unsigned)p_st->tm_mon + 1; - day_in_month_ = (unsigned)p_st->tm_mday; - day_in_year_ = (unsigned)p_st->tm_yday + 1; - day_in_week_ = (unsigned)p_st->tm_wday; - hour_ = (unsigned)p_st->tm_hour; - minute_ = (unsigned)p_st->tm_min; - second_ = (unsigned)p_st->tm_sec; -} - -Time::~Time () -{ - ; -} - -unsigned -Time::getYear (void) const -{ - return(year_); -} - -string -Time::getYear (unsigned digits) const -{ - return( 0 == digits ? - convUnsignedToStr(getYear()) : resizeStringByEndian(convUnsignedToStr(getYear()), digits) ); -} - -unsigned -Time::getMonth (void) const -{ - return(month_); -} - -string -Time::getMonth (unsigned digits) const -{ - return( 0 == digits ? - convUnsignedToStr(getMonth()) : resizeStringByEndian(convUnsignedToStr(getMonth()), digits) ); -} - -unsigned -Time::getDayInWeek (void) const -{ - return(day_in_week_); -} - -string -Time::getDayInWeek (bool b_abbr) const -{ - switch (getDayInWeek()) { - case 1: - return(b_abbr ? "mon" : "monday"); - case 2: - return(b_abbr ? "tues" : "tuesday"); - case 3: - return(b_abbr ? "wed" : "wednesday"); - case 4: - return(b_abbr ? "thurs" : "thursday"); - case 5: - return(b_abbr ? "fri" : "friday"); - case 6: - return(b_abbr ? "sat" : "saturday"); - case 0: - return(b_abbr ? "sun" : "sunday"); - default: - return(""); - } -} - -unsigned -Time::getDayInMonth (void) const -{ - return(day_in_month_); -} - -string -Time::getDayInMonth (unsigned digits) const -{ - return( 0 == digits ? - convUnsignedToStr(getDayInMonth()) : resizeStringByEndian(convUnsignedToStr(getDayInMonth()), digits) ); -} - -unsigned -Time::getDayInYear (void) const -{ - return(day_in_year_); -} - -string -Time::getDayInYear (unsigned digits) const -{ - return( 0 == digits ? - convUnsignedToStr(getDayInYear()) : resizeStringByEndian(convUnsignedToStr(getDayInYear()), digits) ); -} -unsigned -Time::getHour (void) const -{ - return(hour_); -} - -string -Time::getHour (unsigned digits) const -{ - return( 0 == digits ? - convUnsignedToStr(getHour()) : resizeStringByEndian(convUnsignedToStr(getHour()), digits) ); -} - -unsigned -Time::getMinute (void) const -{ - return(minute_); -} - -string -Time::getMinute (unsigned digits) const -{ - return( 0 == digits ? - convUnsignedToStr(getMinute()) : resizeStringByEndian(convUnsignedToStr(getMinute()), digits) ); -} - -unsigned -Time::getSecond (void) const -{ - return(second_); -} - -string -Time::getSecond (unsigned digits) const -{ - return( 0 == digits ? - convUnsignedToStr(getSecond()) : resizeStringByEndian(convUnsignedToStr(getSecond()), digits) ); -} - diff --git a/src/lib/helper/Time.h b/src/lib/helper/Time.h deleted file mode 100644 index d9a7c12..0000000 --- a/src/lib/helper/Time.h +++ /dev/null @@ -1,46 +0,0 @@ -// last modified - -#pragma once -#include - -using std::string; - -class Time -{ - public: - Time (); - virtual ~Time (); - - unsigned getYear (void) const; - string getYear (unsigned digits) const; - - unsigned getMonth (void) const; - string getMonth (unsigned digits) const; - - unsigned getDayInWeek (void) const; - string getDayInWeek (bool b_abbr) const; - unsigned getDayInMonth (void) const; - string getDayInMonth (unsigned digits) const; - unsigned getDayInYear (void) const; - string getDayInYear (unsigned digits) const; - - unsigned getHour (void) const; - string getHour (unsigned digits) const; - - unsigned getMinute (void) const; - string getMinute (unsigned digits) const; - - unsigned getSecond (void) const; - string getSecond (unsigned digits) const; - - private: - unsigned year_; - unsigned month_; - unsigned day_in_month_; - unsigned day_in_week_; - unsigned day_in_year_; - unsigned hour_; - unsigned minute_; - unsigned second_; -}; - diff --git a/src/lib/self/Aicheng.cpp b/src/lib/self/Aicheng.cpp deleted file mode 100644 index 66ce279..0000000 --- a/src/lib/self/Aicheng.cpp +++ /dev/null @@ -1,333 +0,0 @@ -// last modified - -#include "Aicheng.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include "AichengTopicsListWebpage.h" -#include "AichengTopicWebpage.h" -#include "JandownSeedWebpage.h" -#include "../helper/RichTxt.h" - - -using namespace std; - - -static mutex g_mtx; - -static const string& -getPortalWebpageUrl (void) -{ - static const string portal_url("http://www.ac168.info/bt/simple/"); - return(portal_url); -} - -static const string& -getTopicsListWebpagePartUrl (Aicheng::AvClass av_class) -{ - static const string west_part_url("index.php?f5.html"); - static const string cartoon_part_url("index.php?f6.html"); - static const string asia_mosaicked_part_url("index.php?f4.html"); - static const string asia_non_mosaicked_part_url("index.php?f16.html"); - - switch (av_class) { - case Aicheng::west: - return(west_part_url); - case Aicheng::cartoon: - return(cartoon_part_url); - case Aicheng::asia_mosaicked: - return(asia_mosaicked_part_url); - case Aicheng::asia_non_mosaicked: - return(asia_non_mosaicked_part_url); - } -} - -static const string -getTopicsListWebpageUrl (Aicheng::AvClass av_class) -{ - return(getPortalWebpageUrl() + getTopicsListWebpagePartUrl(av_class)); -} - -static bool -isThereInList ( const string& webpage_title, - const vector& ignore_keywords_list, - string& which_keyword ) -{ - for (const auto& e : ignore_keywords_list) { - if (!e.empty() && string::npos != webpage_title.find(e)) { - which_keyword = e; - return(true); - } - } - - return(false); -} - -static bool -parseValidTopicsUrls ( Aicheng::AvClass av_class, - const string& proxy_addr, - unsigned range_begin, unsigned range_end, - const vector& hate_keywords_list, - const vector& like_keywords_list, - vector& valid_topics_urls_list, - bool b_progress ) -{ - valid_topics_urls_list.clear(); - - string current_url = getTopicsListWebpageUrl(av_class); - bool b_stop = false; - unsigned topics_cnt = 0; - while (!current_url.empty() && !b_stop) { - AichengTopicsListWebpage aicheng_topicslist_webpage(current_url, proxy_addr); - if (!aicheng_topicslist_webpage.isLoaded()) { - return(false); - } - - const vector>& topics_title_and_url = aicheng_topicslist_webpage.getTitlesAndUrlsList(); - for (const auto& e : topics_title_and_url) { - if (++topics_cnt > range_end) { - b_stop = true; - break; - } - - const string& topic_title = e.first; - const string& topic_url = e.second; - static const string o_flag(RichTxt::bold_on + "O" + RichTxt::bold_off); - static const string x_flag("x"); - - // ignore the topics which do not in range - if (topics_cnt < range_begin) { - if (b_progress) { - cout << x_flag << " " << flush; - } - continue; - } - // ignore the topics which contain hate keyword by user set - string which_keyword; - if (isThereInList(topic_title, hate_keywords_list, which_keyword)) { - if (b_progress) { - cout << x_flag << " " << flush; - } - continue; - } - // ignore the topics which do not contain like keyword by user set - if ( !like_keywords_list.empty() && - !isThereInList(topic_title, like_keywords_list, which_keyword) ) { - if (b_progress) { - cout << x_flag << " " << flush; - } - continue; - } - - valid_topics_urls_list.push_back(topic_url); - - if (b_progress) { - cout << o_flag << " " << flush; - } - } - - current_url = aicheng_topicslist_webpage.getNextpageUrl(); - } - - - return(true); -} - -static void -downloadTopicPicsAndSeed ( const string& topic_url, - const string& proxy_addr, - const string& path, - unsigned timeout_download_pic, - bool b_show_info ) -{ - AichengTopicWebpage aicheng_topics_webpage(topic_url, proxy_addr); - - // ready for the basename of pictures and seed. - // >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> - string base_name; // from topic title - - // 0) delete the web logo info; - // 1) clear the "/" in topictitle string, if the "/" present in filename, - // linux will treat it as directory, again, clear the "\" for windows; - static const vector keyword_logos_list = { " 亚洲无码区 bt下载 - powered by phpwind.net", - " 亚洲有码区 bt下载 - powered by phpwind.net", - " 欧美区 bt下载 - powered by phpwind.net", - " 动漫区 bt下载 - powered by phpwind.net" }; - const string& topic_webpage_title = aicheng_topics_webpage.getTitle(); - auto keyword_logo_pos = string::npos; - for (const auto& f : keyword_logos_list) { - keyword_logo_pos = topic_webpage_title.find(f); - if (string::npos != keyword_logo_pos) { - break; - } - } - remove_copy_if( topic_webpage_title.cbegin(), - (string::npos == keyword_logo_pos) ? topic_webpage_title.cend() : topic_webpage_title.cbegin() + (int)keyword_logo_pos, - back_inserter(base_name), - [] (char ch) {return('\\' == ch || '/' == ch);} ); - - // 2) the path + filename max length must less than pathconf(, _PC_NAME_MAX) - const unsigned filename_max_length_without_postfix = (unsigned)pathconf(path.c_str(), _PC_NAME_MAX) - - string("99").size() // picture number - - string(".torrent").size(); - if (base_name.size() >= filename_max_length_without_postfix) { - // the filename too long to create file. the way as following doesn't work, case filename encoding error: - // base_name.resize(filename_max_length_without_postfix - 1), because this is string on char not wstring on wchar. - // there is another stupid way, random name from 'a' to 'z' - base_name.resize(16); - generate( base_name.begin(), base_name.end(), - [] () {return('a' + rand() % ('z' - 'a'));} ); - base_name = "(rename)" + base_name; - } - // <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< - - // download all pictures - vector fail_download_pics_urls_list; - bool b_download_pics_success = aicheng_topics_webpage.downloadAllPictures( path, - base_name, - timeout_download_pic, - fail_download_pics_urls_list, - 8 ); - - // download seed - bool b_downloaded_seed_success = false; - if (!aicheng_topics_webpage.getSeedUrl().empty()) { - JandownSeedWebpage jan_seed_webpage(aicheng_topics_webpage.getSeedUrl(), proxy_addr); - b_downloaded_seed_success = jan_seed_webpage.downloadSeed(path, base_name); - } - - // show result info - if (!b_show_info) { - return; - } - static const string success_info("success"); - static const string fail_info = RichTxt::foreground_red + "failure" + RichTxt::reset_all; - g_mtx.lock(); - cout << " \"" << base_name << "\" - "; - if (b_download_pics_success && b_downloaded_seed_success) { - cout << success_info; - } else { - cout << fail_info << " (download error from " << topic_url << ". "; - if (!b_download_pics_success) { - cout << "pictures error: "; - copy(fail_download_pics_urls_list.cbegin(), fail_download_pics_urls_list.cend(), ostream_iterator(cout, ", ")); - cout << "\b\b"; - } - if (!b_downloaded_seed_success) { - if (!b_download_pics_success) { - cout << "; "; - } - cout << "seed error: " << aicheng_topics_webpage.getSeedUrl(); - } - cout << ")"; - } - cout << endl; - g_mtx.unlock(); -} - -static const string& -getNextProxyAddr (const vector& proxy_addrs_list) -{ - if (proxy_addrs_list.empty()) { - static const string empty_str(""); - return(empty_str); - } - - static unsigned current_pos; - if (current_pos >= proxy_addrs_list.size()) { - current_pos = 0; - } - return(proxy_addrs_list[current_pos++]); -} - -Aicheng::Aicheng ( AvClass av_class, - const vector& proxy_addrs_list, - unsigned range_begin, unsigned range_end, - const vector& hate_keywords_list, - const vector& like_keywords_list, - unsigned threads_total, - unsigned timeout_download_pic, - const string& path ) -{ - // parse the URLs of valid topics by: range, hate keywords, like keywords - cout << "Parse the URLs of topics from " << range_begin << " to " << range_end << ": "; - vector valid_topics_urls_list; - parseValidTopicsUrls( av_class, - getNextProxyAddr(proxy_addrs_list), - range_begin, range_end, - hate_keywords_list, - like_keywords_list, - valid_topics_urls_list, - true ); - if (valid_topics_urls_list.empty()) { - cout << "(There is no topic which you like) " << endl; - return; - } - cout << endl << endl; - - // download all pictures and seeds of topics - cout << "Download the pictures and seeds of topics: " << endl; - unsigned parsed_topics_cnt = 0; - for (unsigned i = 0; i < (valid_topics_urls_list.size() / threads_total); ++i) { - vector threads_list; - for (unsigned j = 0; j < threads_total; ++j) { - ++parsed_topics_cnt; - threads_list.push_back(thread( &downloadTopicPicsAndSeed, - ref(valid_topics_urls_list[i * threads_total + j]), - ref(getNextProxyAddr(proxy_addrs_list)), - ref(path), - timeout_download_pic, - true )); - } - for (auto& e : threads_list) { - if (e.joinable()) { - e.join(); - } - } - - if (!threads_list.empty()) { - cout << setprecision(1) << setiosflags(ios::fixed); - cout << " " << RichTxt::bold_on << RichTxt::underline_on << "<---- " - << 100.0 * parsed_topics_cnt / valid_topics_urls_list.size() - << "% ---->" << RichTxt::underline_off << RichTxt::bold_off << endl; - cout << resetiosflags(ios::fixed); - } - } - - vector threads_list; - for (unsigned i = (valid_topics_urls_list.size() / threads_total) * threads_total; i < valid_topics_urls_list.size(); ++i) { - ++parsed_topics_cnt; - threads_list.push_back(thread( &downloadTopicPicsAndSeed, - ref(valid_topics_urls_list[i]), - ref(getNextProxyAddr(proxy_addrs_list)), - ref(path), - timeout_download_pic, - true )); - } - for (auto& e : threads_list) { - if (e.joinable()) { - e.join(); - } - } - if (!threads_list.empty()) { - cout << setprecision(1) << setiosflags(ios::fixed); - cout << " " << RichTxt::bold_on << RichTxt::underline_on << "<---- " - << 100.0 * parsed_topics_cnt / valid_topics_urls_list.size() - << "% ---->" << RichTxt::underline_off << RichTxt::bold_off << endl; - cout << resetiosflags(ios::fixed); - } - - cout << endl; - cout << "Hey kiddo, your hot babes " << path << ", enjoy it! " << endl; -} - -Aicheng::~Aicheng () -{ - ; -} - diff --git a/src/lib/self/Aicheng.h b/src/lib/self/Aicheng.h deleted file mode 100644 index 9ff9a81..0000000 --- a/src/lib/self/Aicheng.h +++ /dev/null @@ -1,28 +0,0 @@ -// last modified - -#pragma once - -#include -#include - -using std::string; -using std::vector; - - -class Aicheng -{ - public: - enum AvClass {west, cartoon, asia_mosaicked, asia_non_mosaicked}; - - public: - Aicheng ( AvClass av_class, - const vector& proxy_addrs_list, - unsigned range_begin, unsigned range_end, - const vector& hate_keywords_list, - const vector& like_keywords_list, - unsigned threads_total, - unsigned timeout_download_pic, - const string& path ); - virtual ~Aicheng (); -}; - diff --git a/src/lib/self/AichengTopicWebpage.cpp b/src/lib/self/AichengTopicWebpage.cpp deleted file mode 100644 index 2a65b3e..0000000 --- a/src/lib/self/AichengTopicWebpage.cpp +++ /dev/null @@ -1,145 +0,0 @@ -// last modified - -#include "AichengTopicWebpage.h" -#include -#include -#include - - -using namespace std; - -static bool -parsePicturesUrlsHelper ( const string& webpage_txt, - vector& pictures_urls_list, - const string& begin_keyword, - const string& end_keyword ) -{ - bool b_ok = false; - - size_t keyword_pic_begin_pos = 0, keyword_pic_end_pos = 0; - while (true) { - // parse picture URL - keyword_pic_begin_pos = webpage_txt.find(begin_keyword, keyword_pic_end_pos); - if (string::npos == keyword_pic_begin_pos) { - break; - } - keyword_pic_end_pos = webpage_txt.find(end_keyword, keyword_pic_begin_pos + begin_keyword.size()); - if (string::npos == keyword_pic_end_pos) { - //cerr << "WARNING! parsePicturesUrlsHelper() CANNOT find the keyword " << end_keyword << endl; - return(false); - } - string pic_url = webpage_txt.substr( keyword_pic_begin_pos + begin_keyword.size(), - keyword_pic_end_pos - keyword_pic_begin_pos - begin_keyword.size() ); - b_ok = true; - - // there are some bad picture-webspaces, ignore them - bool bad_pic_websapce = false; - static const vector bad_pic_websapces_list = {/* "uploadfr.com", "picrar.com" */}; - for (const auto& e : bad_pic_websapces_list) { - if (string::npos != pic_url.find(e)) { - bad_pic_websapce = true; - break; - } - } - if (bad_pic_websapce) { - continue; - } - - // ignore gif, because gifs almost are AD - if (string::npos != pic_url.rfind(".gif")) { - continue; - } - - // convert https to http - static const string keyword_https("https://"); - const auto https_pos = pic_url.find(keyword_https); - if (string::npos != https_pos) { - static const string keyword_http("http://"); - pic_url.replace(https_pos, keyword_https.size(), keyword_http); - } - - // save the picture URL - pictures_urls_list.push_back(pic_url); - } - - return(b_ok); -} - -static bool -parsePicturesUrls (const string& webpage_txt, vector& pictures_urls_list) -{ - pictures_urls_list.clear(); - - static const string begin_keyword0(" keywords_seed_begin_list = { "http://www.jandown.com", - "http://jandown.com", - "http://www6.mimima.com", - "http://mimima.com" }; - auto keyword_seed_begin_pos = string::npos; - bool b_find_begin = false; - for (const auto& e : keywords_seed_begin_list) { - keyword_seed_begin_pos = webpage_txt.find(keyword_prefix + e); - if (string::npos != keyword_seed_begin_pos) { - b_find_begin = true; - break; - } - } - if (!b_find_begin) { - //cerr << "WARNING! parseSeedUrl() cannot find any keywords " << keyword_prefix << " + : "; - //copy( keywords_seed_begin_list.cbegin(), keywords_seed_begin_list.cend(), - //ostream_iterator(cerr, " ") ); - //cerr << endl; - return(false); - } - - static const string keyword_seed_end("\""); - const auto keyword_seed_end_pos = webpage_txt.find( keyword_seed_end, - keyword_seed_begin_pos + keyword_prefix.size() ); - if (string::npos == keyword_seed_end_pos) { - //cerr << "WARNING! parseSeedUrl() cannot find the keyword " << keyword_seed_end << endl; - return(false); - } - - seed_url = webpage_txt.substr( keyword_seed_begin_pos + keyword_prefix.size(), - keyword_seed_end_pos - keyword_seed_begin_pos - keyword_prefix.size() ); - - - return(true); -} - -AichengTopicWebpage::AichengTopicWebpage (const string& url, const string& proxy_addr) - : TopicWebpage(url, parsePicturesUrls, parseSeedUrl, proxy_addr, "gbk", "UTF-8") -{ - ; -} - -AichengTopicWebpage::~AichengTopicWebpage () -{ - ; -} - diff --git a/src/lib/self/AichengTopicWebpage.h b/src/lib/self/AichengTopicWebpage.h deleted file mode 100644 index 884d8b0..0000000 --- a/src/lib/self/AichengTopicWebpage.h +++ /dev/null @@ -1,19 +0,0 @@ -// last modified - -#pragma once - -#include -#include -#include "TopicWebpage.h" - -using std::string; -using std::vector; - - -class AichengTopicWebpage : public TopicWebpage -{ - public: - AichengTopicWebpage (const string& url, const string& proxy_addr); - virtual ~AichengTopicWebpage (); -}; - diff --git a/src/lib/self/AichengTopicsListWebpage.cpp b/src/lib/self/AichengTopicsListWebpage.cpp deleted file mode 100644 index b9f50a5..0000000 --- a/src/lib/self/AichengTopicsListWebpage.cpp +++ /dev/null @@ -1,107 +0,0 @@ -// last modified - -#include "AichengTopicsListWebpage.h" -#include -#include - - -using namespace std; - - -static const string& -getPortalWebpageUrl (void) -{ - static const string portal_url("http://www.ac168.info/bt/"); - return(portal_url); -} - -static bool -parseTitlesAndUrls (const string& webpage_txt, vector>& titles_and_urls_list) -{ - const unsigned size_back = titles_and_urls_list.size(); - - size_t keyword_topic_url_begin_pos = 0, keyword_topic_url_end_pos = 0; - size_t keyword_topic_title_begin_pos = 0, keyword_topic_title_end_pos = 0; - - static const vector rule_topic_keywords_list = { "防止盗号", "版规", "求 片 區"}; // "防止盗号" must in the first, because - // "版规" and "防止盗号" successively appear - // in the same topics list webpage - // http://www.ac168.info/bt/simple/index.php?f16.html - for (const auto& e : rule_topic_keywords_list) { - keyword_topic_url_begin_pos = webpage_txt.find(e); - if (string::npos != keyword_topic_url_begin_pos) { - break; - } - } - if (string::npos == keyword_topic_url_begin_pos) { - keyword_topic_url_begin_pos = 0; - } - - while (true) { - // parse topic URL - static const string keyword_topic_url_begin("

"); - if ( string::npos == (keyword_topic_url_begin_pos = webpage_txt.find(keyword_topic_url_begin, keyword_topic_url_begin_pos)) || - string::npos == (keyword_topic_url_end_pos = webpage_txt.find(keyword_topic_url_end, keyword_topic_url_begin_pos + keyword_topic_url_begin.size())) ) { - break; - } - const string topic_url = getPortalWebpageUrl() + - webpage_txt.substr( keyword_topic_url_begin_pos + keyword_topic_url_begin.size(), - keyword_topic_url_end_pos - keyword_topic_url_begin_pos - keyword_topic_url_begin.size() ); - - // parse topic title - static const string keyword_topic_title_begin("\">"); - static const string keyword_topic_title_end(""); - if ( string::npos == (keyword_topic_title_begin_pos = webpage_txt.find(keyword_topic_title_begin, keyword_topic_url_end_pos)) || - string::npos == (keyword_topic_title_end_pos = webpage_txt.find(keyword_topic_title_end, keyword_topic_title_begin_pos + keyword_topic_title_begin.size())) ) { - break; - } - const string topic_title = webpage_txt.substr( keyword_topic_title_begin_pos + keyword_topic_title_begin.size(), - keyword_topic_title_end_pos - keyword_topic_title_begin_pos - keyword_topic_title_begin.size() ); - keyword_topic_url_begin_pos = keyword_topic_title_end_pos + 1; - - // save url and title of the topic - titles_and_urls_list.push_back(make_pair(topic_title, topic_url)); - } - - return(titles_and_urls_list.size() > size_back); -} - -static bool -parseNextpageUrl (const string& webpage_txt, string& nextpage_url) -{ - nextpage_url.empty(); - - static const string keyword_nextpage_begin("  "); - auto keyword_nextpage_end_pos = webpage_txt.find( keyword_nextpage_end, - keyword_nextpage_begin_pos + keyword_nextpage_begin.size() ); - if (string::npos == keyword_nextpage_end_pos) { - cerr << "WARNING! parseNextpageUrl() can't find " << keyword_nextpage_end << endl; - return(false); - } - - nextpage_url = getPortalWebpageUrl() + - webpage_txt.substr( keyword_nextpage_begin_pos + keyword_nextpage_begin.size(), - keyword_nextpage_end_pos - keyword_nextpage_begin_pos - keyword_nextpage_begin.size() ); - - return(true); -} - -AichengTopicsListWebpage::AichengTopicsListWebpage (const string& url, const string& proxy_addr) - : TopicsListWebpage(url, parseTitlesAndUrls, parseNextpageUrl, proxy_addr, "gbk", "UTF-8") -{ - ; -} - -AichengTopicsListWebpage::~AichengTopicsListWebpage () -{ - ; -} - diff --git a/src/lib/self/AichengTopicsListWebpage.h b/src/lib/self/AichengTopicsListWebpage.h deleted file mode 100644 index 2996dd4..0000000 --- a/src/lib/self/AichengTopicsListWebpage.h +++ /dev/null @@ -1,18 +0,0 @@ -// last modified - -#pragma once - -#include -#include -#include "TopicsListWebpage.h" - -using std::string; -using std::vector; - - -class AichengTopicsListWebpage : public TopicsListWebpage -{ - public: - AichengTopicsListWebpage (const string& url, const string& proxy_addr); - virtual ~AichengTopicsListWebpage (); -}; diff --git a/src/lib/self/Caoliu.cpp b/src/lib/self/Caoliu.cpp deleted file mode 100644 index 5867ab2..0000000 --- a/src/lib/self/Caoliu.cpp +++ /dev/null @@ -1,374 +0,0 @@ -// last modified - -#include "Caoliu.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include "CaoliuTopicsListWebpage.h" -#include "CaoliuTopicWebpage.h" -#include "RmdownSeedWebpage.h" -#include "../helper/RichTxt.h" - - -using namespace std; - - -static mutex g_mtx; - -static const string& -getPortalWebpageUrl (void) -{ - static const string portal_url("http://cl.man.lv/"); - return(portal_url); -} - -static const string& -getTopicsListWebpagePartUrl (Caoliu::AvClass av_class) -{ - // reposted - static const string west_reposted_part_url("thread0806.php?fid=19"); - static const string cartoon_reposted_part_url("thread0806.php?fid=24"); - static const string asia_mosaicked_reposted_part_url("thread0806.php?fid=18"); - static const string asia_non_mosaicked_reposted_part_url("thread0806.php?fid=17"); - - // original - static const string west_original_part_url("thread0806.php?fid=4"); - static const string cartoon_original_part_url("thread0806.php?fid=5"); - static const string asia_mosaicked_original_part_url("thread0806.php?fid=15"); - static const string asia_non_mosaicked_original_part_url("thread0806.php?fid=2"); - - // selfie - static const string selfie_part_url("thread0806.php?fid=16"); - - switch (av_class) { - case Caoliu::west_reposted: - return(west_reposted_part_url); - case Caoliu::cartoon_reposted: - return(cartoon_reposted_part_url); - case Caoliu::asia_mosaicked_reposted: - return(asia_mosaicked_reposted_part_url); - case Caoliu::asia_non_mosaicked_reposted: - return(asia_non_mosaicked_reposted_part_url); - case Caoliu::west_original: - return(west_original_part_url); - case Caoliu::cartoon_original: - return(cartoon_original_part_url); - case Caoliu::asia_mosaicked_original: - return(asia_mosaicked_original_part_url); - case Caoliu::asia_non_mosaicked_original: - return(asia_non_mosaicked_original_part_url); - case Caoliu::selfie: - return(selfie_part_url); - } -} - -static const string -getTopicsListWebpageUrl (Caoliu::AvClass av_class) -{ - return(getPortalWebpageUrl() + getTopicsListWebpagePartUrl(av_class)); -} - -static bool -isThereInList ( const string& webpage_title, - const vector& ignore_keywords_list, - string& which_keyword ) -{ - for (const auto& e : ignore_keywords_list) { - if (!e.empty() && string::npos != webpage_title.find(e)) { - which_keyword = e; - return(true); - } - } - - return(false); -} - -static bool -parseValidTopicsUrls ( Caoliu::AvClass av_class, - const string& proxy_addr, - unsigned range_begin, unsigned range_end, - const vector& hate_keywords_list, - const vector& like_keywords_list, - vector& valid_topics_urls_list, - bool b_progress ) -{ - valid_topics_urls_list.clear(); - - string current_url = getTopicsListWebpageUrl(av_class); - bool b_stop = false; - unsigned topics_cnt = 0; - while (!current_url.empty() && !b_stop) { - CaoliuTopicsListWebpage caoliu_topicslist_webpage(current_url, proxy_addr); - if (!caoliu_topicslist_webpage.isLoaded()) { - return(false); - } - - const vector>& topics_title_and_url = caoliu_topicslist_webpage.getTitlesAndUrlsList(); - for (const auto& e : topics_title_and_url) { - if (++topics_cnt > range_end) { - b_stop = true; - break; - } - - const string& topic_title = e.first; - const string& topic_url = e.second; - static const string o_flag(RichTxt::bold_on + "O" + RichTxt::bold_off); - static const string x_flag("x"); - - // ignore the topics which do not in range - if (topics_cnt < range_begin) { - if (b_progress) { - cout << x_flag << " " << flush; - } - continue; - } - // ignore the topics which contain hate keyword by user set - string which_keyword; - if (isThereInList(topic_title, hate_keywords_list, which_keyword)) { - if (b_progress) { - cout << x_flag << " " << flush; - } - continue; - } - // ignore the topics which do not contain like keyword by user set - if ( !like_keywords_list.empty() && - !isThereInList(topic_title, like_keywords_list, which_keyword) ) { - if (b_progress) { - cout << x_flag << " " << flush; - } - continue; - } - - valid_topics_urls_list.push_back(topic_url); - - if (b_progress) { - cout << o_flag << " " << flush; - } - } - - current_url = caoliu_topicslist_webpage.getNextpageUrl(); - } - - - return(true); -} - -static void -downloadTopicPicsAndSeed ( const string& topic_url, - const string& proxy_addr, - const string& path, - unsigned timeout_download_pic, - unsigned pictures_total, - bool b_download_seed, - bool b_show_info ) -{ - CaoliuTopicWebpage caoliu_topics_webpage(topic_url, proxy_addr); - - // ready for the basename of pictures and seed. - // >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> - string base_name; // from topic title - - // 0) delete the web logo info; - // 1) clear the "/" in topictitle string, if the "/" present in filename, - // linux will treat it as directory, again, clear the "\" for windows; - static const vector keyword_logos_list = {" 草榴社區 - powered by phpwind.net"}; - const string& topic_webpage_title = caoliu_topics_webpage.getTitle(); - auto keyword_logo_pos = string::npos; - for (const auto& f : keyword_logos_list) { - keyword_logo_pos = topic_webpage_title.find(f); - if (string::npos != keyword_logo_pos) { - break; - } - } - remove_copy_if( topic_webpage_title.cbegin(), - (string::npos == keyword_logo_pos) ? topic_webpage_title.cend() : topic_webpage_title.cbegin() + (int)keyword_logo_pos, - back_inserter(base_name), - [] (char ch) {return('\\' == ch || '/' == ch);} ); - - // 2) the path + filename max length must less than pathconf(, _PC_NAME_MAX) - const unsigned filename_max_length_without_postfix = (unsigned)pathconf(path.c_str(), _PC_NAME_MAX) - - string("99").size() // picture number - - string(".torrent").size(); - if (base_name.size() >= filename_max_length_without_postfix) { - // the filename too long to create file. the way as following doesn't work, case filename encoding error: - // base_name.resize(filename_max_length_without_postfix - 1), because this is string on char not wstring on wchar. - // there is another stupid way, random name from 'a' to 'z' - base_name.resize(16); - generate( base_name.begin(), base_name.end(), - [] () {return('a' + rand() % ('z' - 'a'));} ); - base_name = "(rename)" + base_name; - } - // <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< - - // download all pictures -//const vector& urls = caoliu_topics_webpage.getPicturesUrlsList(); -//for (const auto& e : urls) { - ////cout << "------------" << endl; - //cout << caoliu_topics_webpage.getRemoteFiletype(e) << endl; - ////cout << "------------" << endl; -//} - vector fail_download_pics_urls_list; - bool b_download_pics_success = caoliu_topics_webpage.downloadAllPictures( path, - base_name, - timeout_download_pic, - fail_download_pics_urls_list, - pictures_total ); - - // download seed - bool b_downloaded_seed_success = true; - if (b_download_seed) { - b_downloaded_seed_success = false; - if (!caoliu_topics_webpage.getSeedUrl().empty()) { - RmdownSeedWebpage rm_seed_webpage(caoliu_topics_webpage.getSeedUrl(), proxy_addr); - b_downloaded_seed_success = rm_seed_webpage.downloadSeed(path, base_name); - } - } - - // show result info - if (!b_show_info) { - return; - } - static const string success_info("success"); - static const string fail_info = RichTxt::foreground_red + "failure" + RichTxt::reset_all; - g_mtx.lock(); - cout << " \"" << base_name << "\" - "; - if (b_download_pics_success && b_downloaded_seed_success) { - cout << success_info; - } else { - cout << fail_info << " (download error from " << topic_url << ". "; - if (!b_download_pics_success) { - cout << "pictures error: "; - copy(fail_download_pics_urls_list.cbegin(), fail_download_pics_urls_list.cend(), ostream_iterator(cout, ", ")); - cout << "\b\b"; - } - if (b_download_seed && !b_downloaded_seed_success) { - if (!b_download_pics_success) { - cout << "; "; - } - cout << "seed error: " << caoliu_topics_webpage.getSeedUrl(); - } - cout << ")"; - } - cout << endl; - g_mtx.unlock(); -} - -static const string& -getNextProxyAddr (const vector& proxy_addrs_list) -{ - if (proxy_addrs_list.empty()) { - static const string empty_str(""); - return(empty_str); - } - - static unsigned current_pos; - if (current_pos >= proxy_addrs_list.size()) { - current_pos = 0; - } - return(proxy_addrs_list[current_pos++]); -} - -Caoliu::Caoliu ( AvClass av_class, - const vector& proxy_addrs_list, - unsigned range_begin, unsigned range_end, - const vector& hate_keywords_list, - const vector& like_keywords_list, - unsigned threads_total, - unsigned timeout_download_pic, - const string& path ) -{ - // parse the URLs of valid topics by: range, hate keywords, like keywords - cout << "Parse the URLs of topics from " << range_begin << " to " << range_end << ": "; - vector valid_topics_urls_list; - parseValidTopicsUrls( av_class, - getNextProxyAddr(proxy_addrs_list), - range_begin, range_end, - hate_keywords_list, - like_keywords_list, - valid_topics_urls_list, - true ); - cout << endl; - if (valid_topics_urls_list.empty()) { - cout << "There is no topic which you like. " << endl; - return; - } - cout << endl; - - // check just download picutures for dagaier? - unsigned pictures_total = 2; - bool b_download_seed = true; - if (Caoliu::selfie == av_class) { - pictures_total = 128; - b_download_seed = false; - } - - // download all pictures and seeds of topics - cout << "Download the pictures and seeds of topics: " << endl; - unsigned parsed_topics_cnt = 0; - for (unsigned i = 0; i < (valid_topics_urls_list.size() / threads_total); ++i) { - vector threads_list; - for (unsigned j = 0; j < threads_total; ++j) { - ++parsed_topics_cnt; - threads_list.push_back(thread( &downloadTopicPicsAndSeed, - ref(valid_topics_urls_list[i * threads_total + j]), - ref(getNextProxyAddr(proxy_addrs_list)), - ref(path), - timeout_download_pic, - pictures_total, - b_download_seed, - true )); - } - for (auto& e : threads_list) { - if (e.joinable()) { - e.join(); - } - } - - if (!threads_list.empty()) { - cout << setprecision(1) << setiosflags(ios::fixed); - cout << " " << RichTxt::bold_on << RichTxt::underline_on << "<---- " - << 100.0 * parsed_topics_cnt / valid_topics_urls_list.size() - << "% ---->" << RichTxt::underline_off << RichTxt::bold_off << endl; - cout << resetiosflags(ios::fixed); - } - } - - vector threads_list; - for (unsigned i = (valid_topics_urls_list.size() / threads_total) * threads_total; i < valid_topics_urls_list.size(); ++i) { - ++parsed_topics_cnt; - threads_list.push_back(thread( &downloadTopicPicsAndSeed, - ref(valid_topics_urls_list[i]), - ref(getNextProxyAddr(proxy_addrs_list)), - ref(path), - timeout_download_pic, - pictures_total, - b_download_seed, - true )); - } - for (auto& e : threads_list) { - if (e.joinable()) { - e.join(); - } - } - if (!threads_list.empty()) { - cout << setprecision(1) << setiosflags(ios::fixed); - cout << " " << RichTxt::bold_on << RichTxt::underline_on << "<---- " - << 100.0 * parsed_topics_cnt / valid_topics_urls_list.size() - << "% ---->" << RichTxt::underline_off << RichTxt::bold_off << endl; - cout << resetiosflags(ios::fixed); - } - - cout << endl; - cout << "Hey kiddo, your hot babes " << path << ", enjoy it! " << endl; -} - -Caoliu::~Caoliu () -{ - ; -} - diff --git a/src/lib/self/Caoliu.h b/src/lib/self/Caoliu.h deleted file mode 100644 index 9679708..0000000 --- a/src/lib/self/Caoliu.h +++ /dev/null @@ -1,30 +0,0 @@ -// last modified - -#pragma once - -#include -#include - -using std::string; -using std::vector; - - -class Caoliu -{ - public: - enum AvClass { west_reposted, cartoon_reposted, asia_mosaicked_reposted, asia_non_mosaicked_reposted, - west_original, cartoon_original, asia_mosaicked_original, asia_non_mosaicked_original, - selfie }; - - public: - Caoliu ( AvClass av_class, - const vector& proxy_addrs_list, - unsigned range_begin, unsigned range_end, - const vector& hate_keywords_list, - const vector& like_keywords_list, - unsigned threads_total, - unsigned timeout_download_pic, - const string& path ); - virtual ~Caoliu (); -}; - diff --git a/src/lib/self/CaoliuTopicWebpage.cpp b/src/lib/self/CaoliuTopicWebpage.cpp deleted file mode 100644 index 341f9ef..0000000 --- a/src/lib/self/CaoliuTopicWebpage.cpp +++ /dev/null @@ -1,137 +0,0 @@ -// last modified - -#include "CaoliuTopicWebpage.h" -#include -#include -#include - - -using namespace std; - -static bool -parsePicturesUrlsHelper ( const string& webpage_txt, - vector& pictures_urls_list, - const string& begin_keyword, - const string& end_keyword ) -{ - bool b_ok = false; - - size_t keyword_pic_begin_pos = 0, keyword_pic_end_pos = 0; - while (true) { - // parse picture URL - keyword_pic_begin_pos = webpage_txt.find(begin_keyword, keyword_pic_end_pos); - if (string::npos == keyword_pic_begin_pos) { - break; - } - keyword_pic_end_pos = webpage_txt.find(end_keyword, keyword_pic_begin_pos + begin_keyword.size()); - if (string::npos == keyword_pic_end_pos) { - //cerr << "WARNING! parsePicturesUrlsHelper() CANNOT find the keyword " << end_keyword << endl; - return(false); - } - string pic_url = webpage_txt.substr( keyword_pic_begin_pos + begin_keyword.size(), - keyword_pic_end_pos - keyword_pic_begin_pos - begin_keyword.size() ); - b_ok = true; - - // there are some bad picture-webspaces, ignore them - bool bad_pic_websapce = false; - static const vector bad_pic_websapces_list = {/* "uploadfr.com", "picrar.com" */}; - for (const auto& e : bad_pic_websapces_list) { - if (string::npos != pic_url.find(e)) { - bad_pic_websapce = true; - break; - } - } - if (bad_pic_websapce) { - continue; - } - - // ignore gif, because gifs almost are AD - if (string::npos != pic_url.rfind(".gif")) { - continue; - } - - // convert https to http - static const string keyword_https("https://"); - const auto https_pos = pic_url.find(keyword_https); - if (string::npos != https_pos) { - static const string keyword_http("http://"); - pic_url.replace(https_pos, keyword_https.size(), keyword_http); - } - - // save the picture URL - pictures_urls_list.push_back(pic_url); - } - - - return(b_ok); -} - -static bool -parsePicturesUrls (const string& webpage_txt, vector& pictures_urls_list) -{ - pictures_urls_list.clear(); - - static const string begin_keyword0(" keywords_seed_begin_list = { "http://www.rmdown.com/link.php?hash=", - "http://rmdown.com/link.php?hash=", - "http://www.xunfs.com/link.php?hash=", - "http://xunfs.com/link.php?hash=" }; - auto keyword_seed_begin_pos = string::npos; - bool b_find_begin = false; - for (const auto& e : keywords_seed_begin_list) { - keyword_seed_begin_pos = webpage_txt.find(e); - if (string::npos != keyword_seed_begin_pos) { - b_find_begin = true; - break; - } - } - if (!b_find_begin) { - //cerr << "WARNING! parseSeedUrl() cannot find any keywords " << ": "; - //copy( keywords_seed_begin_list.cbegin(), keywords_seed_begin_list.cend(), - //ostream_iterator(cerr, " ") ); - //cerr << endl; - return(false); - } - - static const string keyword_seed_end(""); - const auto keyword_seed_end_pos = webpage_txt.find(keyword_seed_end, keyword_seed_begin_pos); - if (string::npos == keyword_seed_end_pos) { - //cerr << "WARNING! parseSeedUrl() cannot find the keyword " << keyword_seed_end << endl; - return(false); - } - - seed_url = webpage_txt.substr(keyword_seed_begin_pos, keyword_seed_end_pos - keyword_seed_begin_pos); - - - return(true); -} - -CaoliuTopicWebpage::CaoliuTopicWebpage (const string& url, const string& proxy_addr) - : TopicWebpage(url, parsePicturesUrls, parseSeedUrl, proxy_addr, "gbk", "UTF-8") -{ - ; -} - -CaoliuTopicWebpage::~CaoliuTopicWebpage () -{ - ; -} - diff --git a/src/lib/self/CaoliuTopicWebpage.h b/src/lib/self/CaoliuTopicWebpage.h deleted file mode 100644 index 89a04ab..0000000 --- a/src/lib/self/CaoliuTopicWebpage.h +++ /dev/null @@ -1,19 +0,0 @@ -// last modified - -#pragma once - -#include -#include -#include "TopicWebpage.h" - -using std::string; -using std::vector; - - -class CaoliuTopicWebpage : public TopicWebpage -{ - public: - CaoliuTopicWebpage (const string& url, const string& proxy_addr); - virtual ~CaoliuTopicWebpage (); -}; - diff --git a/src/lib/self/CaoliuTopicsListWebpage.cpp b/src/lib/self/CaoliuTopicsListWebpage.cpp deleted file mode 100644 index c60febd..0000000 --- a/src/lib/self/CaoliuTopicsListWebpage.cpp +++ /dev/null @@ -1,101 +0,0 @@ -// last modified - -#include "CaoliuTopicsListWebpage.h" -#include -#include - - -using namespace std; - - -static const string& -getPortalWebpageUrl (void) -{ - static const string portal_url("http://cl.man.lv/"); - return(portal_url); -} - -static bool -parseTitlesAndUrls (const string& webpage_txt, vector>& titles_and_urls_list) -{ - const unsigned size_back = titles_and_urls_list.size(); - - size_t keyword_topic_url_begin_pos = 0, keyword_topic_url_end_pos = 0; - size_t keyword_topic_title_begin_pos = 0, keyword_topic_title_end_pos = 0; - keyword_topic_url_begin_pos = webpage_txt.find("普通主題"); - if (string::npos == keyword_topic_url_begin_pos) { - keyword_topic_url_begin_pos = 0; - } - - while (true) { - // parse topic URL - static const string keyword_topic_url_begin("

"); - static const string keyword_topic_title_end("

"); - if ( string::npos == (keyword_topic_title_begin_pos = webpage_txt.find(keyword_topic_title_begin, keyword_topic_url_end_pos)) || - string::npos == (keyword_topic_title_end_pos = webpage_txt.find(keyword_topic_title_end, keyword_topic_title_begin_pos + keyword_topic_title_begin.size())) ) { - break; - } - const string topic_title = webpage_txt.substr( keyword_topic_title_begin_pos + keyword_topic_title_begin.size(), - keyword_topic_title_end_pos - keyword_topic_title_begin_pos - keyword_topic_title_begin.size() ); - keyword_topic_url_begin_pos = keyword_topic_title_end_pos + 1; - - // save url and title of the topic - titles_and_urls_list.push_back(make_pair(topic_title, topic_url)); - } - - return(titles_and_urls_list.size() > size_back); -} - -static bool -parseNextpageUrl (const string& webpage_txt, string& nextpage_url) -{ - nextpage_url.empty(); - - static const string keyword_nextpage("下一頁"); - static const string keyword_href("href=\'"); - - const auto keyword_nextpage_pos = webpage_txt.find(keyword_nextpage); - if (string::npos == keyword_nextpage_pos) { - return(false); - } - const auto keyword_href_pos = webpage_txt.rfind(keyword_href, keyword_nextpage_pos); - if (string::npos == keyword_href_pos) { - cerr << "WARNING! parseNextpageUrl() cannot find the keyword " << keyword_href << ". " << endl; - return(false); - } - const auto nextpage_suburl_begin_pos = keyword_href_pos + keyword_href.size(); - const auto nextpage_suburl_end_pos = webpage_txt.find("'", nextpage_suburl_begin_pos); - if (string::npos == nextpage_suburl_end_pos) { - cerr << "WARNING! parseNextpageUrl() cannot find the keyword '. " << endl; - return(false); - } - - nextpage_url = getPortalWebpageUrl() + - webpage_txt.substr(nextpage_suburl_begin_pos, nextpage_suburl_end_pos - nextpage_suburl_begin_pos); - - return(true); -} - -CaoliuTopicsListWebpage::CaoliuTopicsListWebpage (const string& url, const string& proxy_addr) - : TopicsListWebpage(url, parseTitlesAndUrls, parseNextpageUrl, proxy_addr, "gbk", "UTF-8") -{ - ; -} - -CaoliuTopicsListWebpage::~CaoliuTopicsListWebpage () -{ - ; -} - diff --git a/src/lib/self/CaoliuTopicsListWebpage.h b/src/lib/self/CaoliuTopicsListWebpage.h deleted file mode 100644 index d6ff2c7..0000000 --- a/src/lib/self/CaoliuTopicsListWebpage.h +++ /dev/null @@ -1,18 +0,0 @@ -// last modified - -#pragma once - -#include -#include -#include "TopicsListWebpage.h" - -using std::string; -using std::vector; - - -class CaoliuTopicsListWebpage : public TopicsListWebpage -{ - public: - CaoliuTopicsListWebpage (const string& url, const string& proxy_addr); - virtual ~CaoliuTopicsListWebpage (); -}; diff --git a/src/lib/self/JandownSeedWebpage.cpp b/src/lib/self/JandownSeedWebpage.cpp deleted file mode 100644 index 6ce635e..0000000 --- a/src/lib/self/JandownSeedWebpage.cpp +++ /dev/null @@ -1,52 +0,0 @@ -// last modified - -#include "JandownSeedWebpage.h" -#include -#include -#include - -using namespace std; - -static bool -parsePostMultiSections ( const string& webpage_txt, - vector>& post_sections_list ) -{ - // parse the code section - static const string& keyword_code_section_begin(""); - const auto keyword_code_section_end_pos = webpage_txt.find( keyword_code_section_end, - keyword_code_section_begin_pos + keyword_code_section_begin.size() ); - if (string::npos == keyword_code_section_end_pos) { - cerr << "WARNING! JandownSeedWebpage::JandownSeedWebpage() CANNOT find the keyword \"" - << keyword_code_section_end << endl; - return(false); - } - const string ref_content = webpage_txt.substr( keyword_code_section_begin_pos + keyword_code_section_begin.size(), - keyword_code_section_end_pos - keyword_code_section_begin_pos - keyword_code_section_begin.size() ); - post_sections_list.push_back(make_pair("code", ref_content)); - - - return(true); -} - -// seed fetch URL. http://www.jandown.com/ and http://www6.mimima.com/ are -// the same one website, on the other word, from http://www.jandown.com/abcd -// download the seed file same as from http://www6.mimima.com/abcd, so, I need -// just ONE fetch URL -JandownSeedWebpage::JandownSeedWebpage (const string& url, const string& proxy_addr) - : SeedWebpage(url, proxy_addr, "http://www.jandown.com/fetch.php", parsePostMultiSections) -{ - ; -} - -JandownSeedWebpage::~JandownSeedWebpage () -{ - ; -} - diff --git a/src/lib/self/JandownSeedWebpage.h b/src/lib/self/JandownSeedWebpage.h deleted file mode 100644 index d1fb04f..0000000 --- a/src/lib/self/JandownSeedWebpage.h +++ /dev/null @@ -1,16 +0,0 @@ -// last modified - -#pragma once - -#include -#include "SeedWebpage.h" - -using std::string; - - -class JandownSeedWebpage : public SeedWebpage -{ - public: - JandownSeedWebpage (const string& url, const string& proxy_addr); - virtual ~JandownSeedWebpage (); -}; diff --git a/src/lib/self/RmdownSeedWebpage.cpp b/src/lib/self/RmdownSeedWebpage.cpp deleted file mode 100644 index 503dddd..0000000 --- a/src/lib/self/RmdownSeedWebpage.cpp +++ /dev/null @@ -1,65 +0,0 @@ -// last modified - -#include "RmdownSeedWebpage.h" -#include -#include -#include - -using namespace std; - -static bool -parsePostMultiSections ( const string& webpage_txt, - vector>& post_sections_list ) -{ - // parse the ref section - static const string& keyword_ref_section(" -#include "SeedWebpage.h" - -using std::string; - - -class RmdownSeedWebpage : public SeedWebpage -{ - public: - RmdownSeedWebpage (const string& url, const string& proxy_addr); - virtual ~RmdownSeedWebpage (); -}; diff --git a/src/lib/self/SeedWebpage.cpp b/src/lib/self/SeedWebpage.cpp deleted file mode 100644 index 371a4c7..0000000 --- a/src/lib/self/SeedWebpage.cpp +++ /dev/null @@ -1,46 +0,0 @@ -// last modified - -#include "SeedWebpage.h" -#include -#include -#include - -using namespace std; - -SeedWebpage::SeedWebpage ( const string& url, - const string& proxy_addr, - const string& post_url, - ParsePostMultiSections parsePostMultiSections ) - : Webpage(url, "", proxy_addr), post_url_(post_url) -{ - if (!isLoaded()) { - cerr << "WARNING! SeedWebpage::SeedWebpage() CANNOT load webpage \"" - << url << "\"" << endl; - return; - } - - // parse the post method multi sections - parsePostMultiSections(getTxt(), post_sections_list_); -} - -SeedWebpage::~SeedWebpage () -{ - ; -} - -// this is a multipart/formdata style HTTP post method -bool -SeedWebpage::downloadSeed (const string& path, const string& base_name) -{ - if (post_sections_list_.empty()) { - return(false); - } - - // make seed name - static const string seed_postfix(".torrent"); - string seed_filename = path + "/" + base_name + seed_postfix; - - - return(submitMultiPost(post_url_, seed_filename, post_sections_list_)); -} - diff --git a/src/lib/self/SeedWebpage.h b/src/lib/self/SeedWebpage.h deleted file mode 100644 index 9685518..0000000 --- a/src/lib/self/SeedWebpage.h +++ /dev/null @@ -1,32 +0,0 @@ -// last modified - -#pragma once - -#include -#include -#include "Webpage.h" - -using std::string; -using std::vector; -using std::pair; - - -class SeedWebpage : public Webpage -{ - public: - // callback function for parse the multi sections of post - typedef bool (*ParsePostMultiSections) ( const string& webpage_txt, - vector>& post_sections_list ); - - public: - SeedWebpage ( const string& url, - const string& proxy_addr, - const string& post_url, - ParsePostMultiSections parsePostMultiSections ); - virtual ~SeedWebpage (); - bool downloadSeed (const string& path, const string& base_name); - - private: - const string post_url_; - vector> post_sections_list_; -}; diff --git a/src/lib/self/TopicWebpage.cpp b/src/lib/self/TopicWebpage.cpp deleted file mode 100644 index 5600d02..0000000 --- a/src/lib/self/TopicWebpage.cpp +++ /dev/null @@ -1,99 +0,0 @@ -// last modified - -#include "TopicWebpage.h" -#include -#include -#include "../helper/Misc.h" - -#include - -using namespace std; - - -TopicWebpage::TopicWebpage ( const string& url, - ParsePicturesUrls parsePicturesUrls, - ParseSeedUrl parseSeedUrl, - const string& proxy_addr, - const string& src_charset, - const string& dest_charset ) - : Webpage(url, "", proxy_addr) -{ - if (!isLoaded()) { - return; - } - - // charset convert - if (!src_charset.empty() && !dest_charset.empty()) { - convertCharset(src_charset, dest_charset); - } - - // parse the URLs of av pictures - if (!parsePicturesUrls(getTxt(), pictures_urls_list_)) { - //cerr << "WARNING! parsePicturesUrls() failure from " << url << endl; - ; - } - - // parse the URLs of seed - if (!parseSeedUrl(getTxt(), seed_url_)) { - //cerr << "WARNING! parseSeedUrl() failure from " << url << endl; - ; - } -} - -TopicWebpage::~TopicWebpage () -{ - ; -} - -const vector& -TopicWebpage::getPicturesUrlsList (void) const -{ - return(pictures_urls_list_); -} - -const string& -TopicWebpage::getSeedUrl (void) const -{ - return(seed_url_); -} - -// the name rule of pictures: topictitle0.jpg, topictitle1.jpg, topictitle[x].jpg -bool -TopicWebpage::downloadAllPictures ( const string& path, - const string& base_name, - unsigned timeout_download_pic, - vector& fail_download_pics_urls_list, - unsigned pictures_max_num ) -{ - fail_download_pics_urls_list.clear(); - - for ( unsigned i = 0, sucess_cnt = 0; - i < pictures_urls_list_.size() && sucess_cnt < pictures_max_num; - ++i ) { - const string& picture_url = pictures_urls_list_[i]; - - // make picture filename - const string tmp = getRemoteFiletype(picture_url); - static const string keyword("image/"); - string postfix_name; - const auto pos = tmp.find(keyword); - if (string::npos != pos) { - postfix_name = tmp.substr(pos + keyword.size()); - } - - // download pic - const string pic_filename = path + "/" + base_name + "-" + convNumToStr(i) + "." + postfix_name; - if (downloadFile(picture_url, pic_filename, "", timeout_download_pic)) { - ++sucess_cnt; - continue; - } - - //cerr << "WARNING! CANNOT download " << pictures_urls_list_[i] << endl; - fail_download_pics_urls_list.push_back(pictures_urls_list_[i]); - remove(pic_filename.c_str()); - } - - - return(fail_download_pics_urls_list.empty()); -} - diff --git a/src/lib/self/TopicWebpage.h b/src/lib/self/TopicWebpage.h deleted file mode 100644 index 56e06c9..0000000 --- a/src/lib/self/TopicWebpage.h +++ /dev/null @@ -1,41 +0,0 @@ -// last modified - -#pragma once - -#include -#include -#include "Webpage.h" - -using std::string; -using std::vector; - - -class TopicWebpage : public Webpage -{ - public: - // callback function for parse the URLs of av pictures - typedef bool (*ParsePicturesUrls) (const string& webpage_txt, vector& pictures_urls_list); - // callback function for parse the URLs of seed - typedef bool (*ParseSeedUrl) (const string& webpage_txt, string& seed_url); - - public: - TopicWebpage ( const string& url, - ParsePicturesUrls parsePicturesUrls, - ParseSeedUrl parseSeedUrl, - const string& proxy_addr, - const string& src_charset, - const string& dest_charset ); - virtual ~TopicWebpage (); - const string& getSeedUrl (void) const; - const vector& getPicturesUrlsList (void) const; - bool downloadAllPictures ( const string& path, - const string& base_name, - unsigned timeout_download_pic, - vector& fail_download_pics_urls_list, - unsigned pictures_max_num = 32 ); - - private: - string seed_url_; - vector pictures_urls_list_; -}; - diff --git a/src/lib/self/TopicsListWebpage.cpp b/src/lib/self/TopicsListWebpage.cpp deleted file mode 100644 index c49b691..0000000 --- a/src/lib/self/TopicsListWebpage.cpp +++ /dev/null @@ -1,58 +0,0 @@ -// last modified - -#include "TopicsListWebpage.h" -#include - - -using namespace std; - - -TopicsListWebpage::TopicsListWebpage ( const string& url, - TopicsListWebpage::ParseTitlesAndUrls parseTitlesAndUrls, - TopicsListWebpage::ParseNextpageUrl parseNextpageUrl, - const string& proxy_addr, - const string& src_charset, - const string& dest_charset ) - : Webpage(url, "", proxy_addr, 16, 4, 4) -{ - if (!isLoaded()) { - return; - } - - // charset convert - if (!src_charset.empty() && !dest_charset.empty()) { - convertCharset(src_charset, dest_charset); - } - - // parse the URLs and titles of all topics on topicslist webpage - const string& webpage_txt = getTxt(); - parseTitlesAndUrls(webpage_txt, titles_and_urls_list_); - - // unescape html for title - for (auto& e : titles_and_urls_list_) { - string& title = e.first; - title = unescapeHtml(title); - } - - // parse the next topicslist webpage URL - parseNextpageUrl(webpage_txt, nextpage_url_); -} - -TopicsListWebpage::~TopicsListWebpage () -{ - ; -} - -// if there is no more topicslist page, return empty string -const string& -TopicsListWebpage::getNextpageUrl (void) const -{ - return(nextpage_url_); -} - -// first title, second url -const vector>& -TopicsListWebpage::getTitlesAndUrlsList (void) const -{ - return(titles_and_urls_list_); -} diff --git a/src/lib/self/TopicsListWebpage.h b/src/lib/self/TopicsListWebpage.h deleted file mode 100644 index 0c29465..0000000 --- a/src/lib/self/TopicsListWebpage.h +++ /dev/null @@ -1,38 +0,0 @@ -// last modified - -#pragma once - -#include -#include -#include "Webpage.h" - -using std::string; -using std::vector; -using std::pair; - - -class TopicsListWebpage : public Webpage -{ - public: - // callback function for parse titles and URLs of topics - typedef bool (*ParseTitlesAndUrls) ( const string& webpage_txt, - vector>& titles_and_urls_list ); - // callback function for parse next topicslist URL - typedef bool (*ParseNextpageUrl) ( const string& webpage_txt, - string& nextpage_url ); - - public: - TopicsListWebpage ( const string& url, - ParseTitlesAndUrls parseTitlesAndUrls, - ParseNextpageUrl parseNextpageUrl, - const string& proxy_addr = "", - const string& src_charset = "", - const string& dest_charset = "" ); - virtual ~TopicsListWebpage (); - const vector>& getTitlesAndUrlsList (void) const; - const string& getNextpageUrl (void) const; - - private: - vector> titles_and_urls_list_; - string nextpage_url_; -}; diff --git a/src/lib/self/Webpage.cpp b/src/lib/self/Webpage.cpp deleted file mode 100644 index 12d0299..0000000 --- a/src/lib/self/Webpage.cpp +++ /dev/null @@ -1,601 +0,0 @@ -// last modified - -#include "Webpage.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "../helper/Misc.h" - - -using namespace std; - - -static bool -checkErrLibcurl (const CURLcode curl_code, const char* libcurl_err_info, bool b_exit = false, bool b_show_err_info = false) -{ - bool b_success = true; - if (CURLE_OK != curl_code) { - if (b_show_err_info) { - cerr << "WARNING! " << libcurl_err_info << endl; - } - b_success = false; - if (b_exit) { - exit(EXIT_FAILURE); - } - } - return(b_success); -} - -static CURL* -initLibcurl (const char* libcurl_err_info_buff) -{ - CURL* p_curl = nullptr; - - // libcurl global and easy init - if (nullptr == (p_curl = curl_easy_init())) { - cerr << "ERROR! cannot easy init libcurl. " << endl; - exit(EXIT_FAILURE); - } - - // the libcurl error info buffer - if (CURLE_OK != curl_easy_setopt(p_curl, CURLOPT_ERRORBUFFER, libcurl_err_info_buff)) { - cerr << "ERROR! " << libcurl_err_info_buff << endl; - exit(EXIT_FAILURE); - } - - // follow the URL redirect - checkErrLibcurl(curl_easy_setopt(p_curl, CURLOPT_FOLLOWLOCATION, true), libcurl_err_info_buff); - - // to allow multi-threaded unix applications to still set/use all timeout options etc, without risking getting signals - checkErrLibcurl(curl_easy_setopt(p_curl, CURLOPT_NOSIGNAL, true), libcurl_err_info_buff); - - // pretend as firefox browser - checkErrLibcurl( curl_easy_setopt(p_curl, CURLOPT_USERAGENT, "Mozilla/5.0 (X11; Linux i686; rv:30.0) Gecko/20100101 Firefox/30.0"), - libcurl_err_info_buff ); - - // automatically set the Referer to redirect source - checkErrLibcurl(curl_easy_setopt(p_curl, CURLOPT_AUTOREFERER, true), libcurl_err_info_buff); - - //// display libcurl action info - //checkErrLibcurl(curl_easy_setopt(p_curl, CURLOPT_VERBOSE, true), libcurl_err_info_buff); - - - return(p_curl); -} - -static void -cleanupLibcul (CURL* p_curl) -{ - curl_easy_cleanup(p_curl); -} - -static string -makeRandomFilename (void) -{ -#ifdef CYGWIN - return("c:\\hardseed_tmp_" + convNumToStr(getpid()) + convNumToStr(pthread_self())); -#else - return("/tmp/hardseed_tmp_" + convNumToStr(getpid()) + convNumToStr(pthread_self())); -#endif -} - -static string -parseTitle (const string& webpage_txt) -{ - static const string keyword_title_begin(""); - const auto keyword_title_end_pos = webpage_txt.find(keyword_title_end, keyword_title_begin_pos); - if (string::npos == keyword_title_end_pos) { - cerr << "WARNING! parseTitle() CANNOT find the keyword _" << keyword_title_end << "_" << endl; - return(""); - } - - - return(webpage_txt.substr( keyword_title_begin_pos + keyword_title_begin.size(), - keyword_title_end_pos - keyword_title_begin_pos - keyword_title_begin.size() )); -} - -// some chars invalid in URL string, such as, space char, chinese char, so I have to -// escape to legal URL. -// OK, curl_easy_escape() likely do this job, but it's stupid: the function converts -// all input characters that are not a-z, A-Z, 0-9, '-', '.', '_' or '~' to their -// "URL escaped" version (%NN where NN is a two-digit hexadecimal number), in the other -// words, it always convert ':', '/', and '?', that's a bad news. -// This is my way to escape URL: -// 0) split one raw URL string to more sub-string by token chars, such as ':', '/', '=' -// and '?' (may be more); -// 1) escape all sub-str by curl_easy_escape(); -// 2) splice escaped sub-string to URL. -string -Webpage::escapeUrl (const string& raw_url) const -{ - // split one raw URL string to more sub-string by token chars - static const string tokens_list(":/=?&,;"); - vector splited_substr_list; - vector appeared_tokens_list; - splitStr(raw_url, tokens_list, splited_substr_list, appeared_tokens_list); - - // escape all sub-string - vector escaped_substr_list; - for (const auto& e : splited_substr_list) { - char* p_str_escaped = curl_easy_escape(p_curl_, e.c_str(), 0); - if (nullptr == p_str_escaped) { - cerr << "WARNING! " << libcurl_err_info_buff_ << endl; - } - escaped_substr_list.push_back(p_str_escaped); - curl_free(p_str_escaped); - } - - // splice escaped sub-string to URL - string url_escaped; - bool b_tokens_first = (string::npos != tokens_list.find(raw_url[0])); - for (unsigned i = 0; i < escaped_substr_list.size() && i < appeared_tokens_list.size(); ++i) { - if (b_tokens_first) { - url_escaped += appeared_tokens_list[i] + escaped_substr_list[i]; - } else { - url_escaped += escaped_substr_list[i] + appeared_tokens_list[i]; - } } - if (escaped_substr_list.size() > appeared_tokens_list.size()) { - url_escaped += escaped_substr_list[escaped_substr_list.size() - 1]; - } - - - return(url_escaped); -} - -string -Webpage::unescapeHtml (const string& raw_txt) const -{ - string unescaped_html_str = raw_txt; - - // more see http://www.theukwebdesigncompany.com/articles/entity-escape-characters.php - static const vector> escaped_html_chars_list = { make_pair("'", "'"), - make_pair(""", "\""), - make_pair("&", "&"), - make_pair("<", "<"), - make_pair(">", ">"), - make_pair(" ", " "), - make_pair(" ", " "), - make_pair("・", "・"), - make_pair("◯", "â—¯"), - make_pair("♪", "♪"), - make_pair("'", "'") }; - - bool b_find; - size_t pos = 0; - do { - b_find = false; - for (const auto& e : escaped_html_chars_list) { - pos = 0; - const string& escaped_str = e.first, unescaped_str = e.second; - pos = unescaped_html_str.find(escaped_str, pos); - if (string::npos != pos) { - unescaped_html_str.replace(pos, escaped_str.size(), unescaped_str); - b_find = true; - break; - } - } - } while (b_find); - - return(unescaped_html_str); -} - -string -Webpage::requestHttpHeader_ (const string& url, HttpHeader_ header_item) const -{ - const string random_filename = makeRandomFilename(); - FILE* fs_http_header = fopen(random_filename.c_str(), "w"); - if (nullptr == fs_http_header) { - return(""); - } - - // get HTTP header - checkErrLibcurl(curl_easy_setopt(p_curl_, CURLOPT_URL, url.c_str()), libcurl_err_info_buff_); - checkErrLibcurl(curl_easy_setopt(p_curl_, CURLOPT_PROXY, proxy_addr_.c_str()), libcurl_err_info_buff_); - checkErrLibcurl(curl_easy_setopt(p_curl_, CURLOPT_NOBODY, true), libcurl_err_info_buff_); // just request the HTTP header - checkErrLibcurl(curl_easy_setopt(p_curl_, CURLOPT_WRITEHEADER, fs_http_header), libcurl_err_info_buff_); - checkErrLibcurl(curl_easy_perform(p_curl_), libcurl_err_info_buff_); - - // reset, I.E., request HTTP body - checkErrLibcurl(curl_easy_setopt(p_curl_, CURLOPT_NOBODY, false), libcurl_err_info_buff_); // just request the HTTP header - checkErrLibcurl(curl_easy_setopt(p_curl_, CURLOPT_WRITEHEADER, nullptr), libcurl_err_info_buff_); - fclose(fs_http_header); - - // load the file to memory, and parse HTTP header info - string http_header, remote_filename, remote_filesize, remote_filetype, remote_filetime; - static const vector httpheader_keywords_list = { "Content-Type: ", // file type - "Content-Length: ", // file size - "Content-Disposition: ", // file name - "Last-Modified: " }; // file modified time - ifstream ifs(random_filename); - string line; - while (getline(ifs, line)) { - // load to memory - http_header += (line + '\n'); - - // parse HTTP header item - for (const auto& e : httpheader_keywords_list) { - auto pos = line.find(e); - if (string::npos == pos) { - continue; - } - - line.pop_back(); // the last char in line is '\r' - const string& tmp = line.substr(pos + e.size(), string::npos); - if ("Content-Type: " == e) { - remote_filetype = tmp; - } else if ("Content-Length: " == e) { - remote_filesize = tmp; - } else if ("Content-Disposition: " == e) { - remote_filename = tmp; - } else if ("Last-Modified: " == e) { - remote_filetime = tmp; - } - } - } - ifs.close(); - - // return http header item - switch (header_item) { - case header: - return(http_header); - case name: - return(remote_filename); - case type: - return(remote_filetype); - case length: - return(remote_filesize); - case modified: - return(remote_filetime); - } -} - -string -Webpage::getHttpHeader (const string& url) const -{ - return(requestHttpHeader_(url, header)); -} - -string -Webpage::getRemoteFiletype (const string& url) const -{ - return(requestHttpHeader_(url, type)); -} - -string -Webpage::getRemoteFilename (const string& url) const -{ - return(requestHttpHeader_(url, name)); -} - -string -Webpage::getRemoteFilesize (const string& url) const -{ - return(requestHttpHeader_(url, length)); -} - -string -Webpage::getRemoteFiletime (const string& url) const -{ - return(requestHttpHeader_(url, modified)); -} - -// once maked Webpage obj, the page in memory, but if filename is not empty, -// it will saveas file, otherwise, no file. -// notice, no https -Webpage::Webpage ( const string& url, - const string& filename, - const string& proxy_addr, - const unsigned timeout_second, - const unsigned retry_times, - const unsigned retry_sleep_second ) - : p_curl_(initLibcurl(libcurl_err_info_buff_)), url_(url), proxy_addr_(proxy_addr), b_loaded_ok_(false) -{ - // set proxy. - // proxy_addr is made up of protocol, IP and port. - // [protocol://][IP][:port], e.g., - // http://127.0.0.1:8087, this is for GoAgent proxy; - // socks4://127.0.0.1:7070, this is for ssh proxy (yes, the port is setted by ssh -D); - // protocol support as follow: http, https, socks4, socks4a, socks5, socks5h. - // if proxy_addr is "", disable proxy - checkErrLibcurl(curl_easy_setopt(p_curl_, CURLOPT_PROXY, proxy_addr_.c_str()), libcurl_err_info_buff_); - - // don't download none-webpage file when construct the obj - if ("text/html" != getRemoteFiletype(url_)) { - return; - } - - // download webpage to local file - const string& localfile = (filename.empty() ? makeRandomFilename() : filename); - b_loaded_ok_ = download_(url_, localfile, "", timeout_second, retry_times, retry_sleep_second); - if (!b_loaded_ok_) { - if (filename.empty()) { - remove(localfile.c_str()); - } - return; - } - - // read webpage file into string - ifstream ifs(localfile); - string line; - txt_.clear(); - while (getline(ifs, line)) { - txt_ += (line + '\n'); - } - ifs.close(); - - // the caller don't care the loacl file, so delete it - if (filename.empty()) { - remove(localfile.c_str()); - } - - // parse title - title_ = unescapeHtml(parseTitle(txt_)); -} - -Webpage::~Webpage () -{ - cleanupLibcul(p_curl_); -} - -// why same as download_()? -// for future -bool -Webpage::downloadFile ( const string& url, - const string& filename, - const string& referer, - const unsigned timeout_second, - const unsigned retry_times, - const unsigned retry_sleep_second ) -{ - return(download_( url, - filename, - referer, - timeout_second, - retry_times, - retry_sleep_second )); -} - -long -Webpage::parseLatestHttpStatusCode_ (void) -{ - checkErrLibcurl(curl_easy_getinfo(p_curl_, CURLINFO_RESPONSE_CODE, &latest_http_status_code_), libcurl_err_info_buff_); - return(latest_http_status_code_); -} - -long -Webpage::getLatestHttpStatusCode (void) const -{ - return(latest_http_status_code_); -} - -bool -Webpage::isValidLatestHttpStatusCode (void) const -{ - // HTTP status code, 4XX stand for client error, 5XX for server error, - // More info http://en.wikipedia.org/wiki/List_of_HTTP_status_codes - static const char client_error_code = '4', server_error_code = '5'; - const string& latest_http_status_code_str = convNumToStr(latest_http_status_code_); - return( client_error_code != latest_http_status_code_str[0] && - server_error_code != latest_http_status_code_str[0] ); -} - -// failure, return -1 -static long -getFileSize (FILE* fs) -{ - // backup current offset - long offset_bak = ftell(fs); - - // get the filesize - fseek(fs, 0, SEEK_END); - long file_size = ftell(fs); - - // restore current offset - fseek(fs, offset_bak, SEEK_SET); - - - return(file_size); -} - -// download internet file to local file, such as, webpage, picture, mp3, etc. -// note: URL must be http, CANNOT be https -bool -Webpage::download_ ( const string& raw_url, - const string& filename, - const string& referer, - const unsigned timeout_second, - const unsigned retry_times, - const unsigned retry_sleep_second ) -{ - // deal with raw URL, first unescape html, second escape URL - const string url = escapeUrl(unescapeHtml(raw_url)); - - // set the target URL - curl_easy_setopt(p_curl_, CURLOPT_URL, url.c_str()); - - // set the referer page. - // note curl_easy_setopt(p_curl_, CURLOPT_REFERER, "") still send referer - // in HTTP header (empty string ""), if you don't wanna sent referer info, - // never touch CURLOPT_REFERER. - if (!referer.empty()) { - curl_easy_setopt(p_curl_, CURLOPT_REFERER, referer.c_str()); - } - - // timeout - checkErrLibcurl(curl_easy_setopt(p_curl_, CURLOPT_TIMEOUT, timeout_second), libcurl_err_info_buff_); - - // ready for downloading webpage to locale tmp file - FILE* fs = fopen(filename.c_str(), "w+"); - if (nullptr == fs) { - cerr << "ERROR! Webpage::download() something happened. Fail to open file " << filename << endl; - return(false); - } - checkErrLibcurl(curl_easy_setopt(p_curl_, CURLOPT_WRITEDATA, fs), libcurl_err_info_buff_); - - // download the URL webpage to locale file - bool b_downloaded = false; - for (unsigned i = 0; i < retry_times; ++i) { - bool b_ok = checkErrLibcurl(curl_easy_perform(p_curl_), libcurl_err_info_buff_); - latest_http_status_code_ = parseLatestHttpStatusCode_(); // notice, parseLatestHttpStatusCode_() must - // very close every curl_easy_perform(p_curl_) - - // notice, even though there is HTTP request status error, libcurl still download - // web file success, of course, this is not real success, so, I have to check the - // http status code - if (b_ok && isValidLatestHttpStatusCode() && getFileSize(fs) > 0) { - b_downloaded = true; - break; - } - - //cerr << "WARNING! Webpage::download() something happened. Fail to download " << url - //<< ", sleeping " << retry_sleep_second << " seconds will retry " << i + 1 << ". " << flush; - - // if libcurl download internet file failure, - // the next retry will append data to local file. - // so, I must clear the filestream by freopen(), - // and, neither rewind() nor fseek(fs, 0L, SEEK_SET) works. - fs = freopen(filename.c_str(), "w+", fs); - if (nullptr == fs) { - cerr << "ERROR! Webpage::download() something happened. Fail to reopen file " << filename << endl; - fclose(fs); - return(false); - } - sleep(retry_sleep_second); - } - fclose(fs); - - - return(b_downloaded); -} - -bool -Webpage::submitMultiPost ( const string& url, - const string& filename, - const vector>& post_sections_list, - const unsigned timeout_second, - const unsigned retry_times, - const unsigned retry_sleep_second) -{ - // construct the all sections for multipart/formdata style HTTP post - // >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> - struct curl_httppost* p_first_section = nullptr; - struct curl_httppost* p_last_section = nullptr; - - for (const auto& e : post_sections_list) { - const string& name = e.first; - const string& content = e.second; - - if (curl_formadd( &p_first_section, &p_last_section, - CURLFORM_PTRNAME, name.c_str(), - CURLFORM_PTRCONTENTS, content.c_str(), - CURLFORM_END )) { - //cerr << "ERROR! RmdownSeedDownloadWebpage::downloadSeed() call curl_formadd() failure! " << endl; - return(false); - } - } - - if (!checkErrLibcurl(curl_easy_setopt(p_curl_, CURLOPT_HTTPPOST, p_first_section), libcurl_err_info_buff_)) { - //cerr << "ERROR! RmdownSeedDownloadWebpage::downloadSeed() call curl_easy_setopt() failure! " << endl; - return(false); - } - // <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< - - bool b_success = download_(url, filename, url_, timeout_second, retry_times, retry_sleep_second); - - // TODO. reset CURLOPT_HTTPPOST, are you sure it works? - static struct curl_httppost empty_httppost; - curl_easy_setopt(p_curl_, CURLOPT_HTTPPOST, &empty_httppost); - - - return(b_success); -} - -const string& -Webpage::getTxt (void) const -{ - return(txt_); -} - -bool -Webpage::isLoaded (void) const -{ - return(b_loaded_ok_); -} - -bool -Webpage::saveasFile (const string& filename) const -{ - ofstream ofs_webpage(filename); - istringstream iss_webpage_txt(txt_); - - string line; - while (getline(iss_webpage_txt, line)) { - ofs_webpage << line << "\n"; - } - - ofs_webpage.close(); - - return(true); -} - -size_t -Webpage::convertCharset (const string& src_charset, const string& dest_charset) -{ - string dest_charset_str; - - istringstream iss_webpage_txt(txt_); - string line; - iconv_t cd = iconv_open((dest_charset + "//IGNORE").c_str(), src_charset.c_str()); // why "//IGNORE"? When the string "//IGNORE" is - // appended to dest char set, some char that cannot - // be represented in the target character set will - // be silently discarded, ohterwise iconv() will - // stop at this char - while (getline(iss_webpage_txt, line)) { - line += "\n"; - size_t inbytes_left = line.size(); - char* p_inbuff = (char*)line.c_str(); - size_t outbytes_left = 2 * inbytes_left; // TODO. why 2? utf8 use 3 bytes for one asia character, 1 byte for one west wcharacter; gbk use 2 bytes for everyone character, so double is the guard value - char* outbuff = new char [outbytes_left]; - memset(outbuff, '\0', outbytes_left); - char* p_outbuff = outbuff; - - if ((size_t)-1 == iconv(cd, &p_inbuff, &inbytes_left, &p_outbuff, &outbytes_left)) { - //cerr << "WARNING! iconv() "<< strerror(errno) << endl; - ; - } - dest_charset_str += outbuff; - - delete [] outbuff; - } - iconv_close(cd); - - txt_ = dest_charset_str; - - title_ = unescapeHtml(parseTitle(txt_)); - - return(txt_.size()); -} - -const string& -Webpage::getTitle (void) const -{ - return(title_); -} - diff --git a/src/lib/self/Webpage.h b/src/lib/self/Webpage.h deleted file mode 100644 index a1ce7fc..0000000 --- a/src/lib/self/Webpage.h +++ /dev/null @@ -1,80 +0,0 @@ -// last modified - -#pragma once - -#include -#include -#include - -using std::string; -using std::vector; -using std::pair; - -class Webpage -{ - public: - explicit Webpage ( const string& url, - const string& filename = "", - const string& proxy_addr = "", - const unsigned timeout_second = 16, - const unsigned retry_times = 2, - const unsigned retry_sleep_second = 2 ); - virtual ~Webpage (); - - const string& getTxt (void) const; - const string& getTitle (void) const; - //const string& getUrl (void) const; - - long getLatestHttpStatusCode (void) const; - bool isValidLatestHttpStatusCode (void) const; - - string getHttpHeader (const string& url) const; - string getRemoteFiletype (const string& url) const; - string getRemoteFilesize (const string& url) const; - string getRemoteFilename (const string& url) const; - string getRemoteFiletime (const string& url) const; - - bool isLoaded (void) const; - size_t convertCharset (const string& src_charset, const string& dest_charset); - bool saveasFile (const string& filename) const; - - bool downloadFile ( const string& url, - const string& filename, - const string& referer = "", - const unsigned timeout_second = 0, - const unsigned retry_times = 2, - const unsigned retry_sleep_second = 2); - bool submitMultiPost ( const string& url, - const string& filename, - const vector>& post_sections_list, - const unsigned timeout_second = 32, - const unsigned retry_times = 2, - const unsigned retry_sleep_second = 2 ); - - string escapeUrl (const string& raw_url) const; - string unescapeHtml (const string& raw_txt) const; - - private: - bool download_ ( const string& raw_url, - const string& filename, - const string& referer, - const unsigned timeout_second, - const unsigned retry_times, - const unsigned retry_sleep_second ); - long parseLatestHttpStatusCode_ (void); - - private: - enum HttpHeader_ {header, type, length, name, modified}; - string requestHttpHeader_ (const string& url, HttpHeader_ header_item) const; - - private: - CURL* p_curl_; - string url_; - char libcurl_err_info_buff_[CURL_ERROR_SIZE]; - string proxy_addr_; - string txt_; - string title_; - bool b_loaded_ok_; - long latest_http_status_code_; -}; - diff --git a/src/main.cpp b/src/main.cpp deleted file mode 100644 index 6712b7d..0000000 --- a/src/main.cpp +++ /dev/null @@ -1,469 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "lib/self/Webpage.h" -#include "lib/self/Aicheng.h" -#include "lib/self/Caoliu.h" -#include "lib/helper/Time.h" -#include "lib/helper/CmdlineOption.h" -#include "lib/helper/RichTxt.h" -#include "lib/helper/Misc.h" - - -using namespace std; - - -static const string g_softname(RichTxt::bold_on + "hardseed" + RichTxt::bold_off); -static const string g_version("0.2.0"); -static const string g_myemail("yangyang.gnu@gmail.com"); -static const string g_myemail_color(RichTxt::bold_on + RichTxt::foreground_green + g_myemail + RichTxt::reset_all); -static const string g_mywebspace("http://www.yangyangwithgnu.net/"); -static const string g_mywebspace_color(RichTxt::bold_on + RichTxt::foreground_green + g_mywebspace + RichTxt::reset_all); - - -static void -showSexyGirl (void) -{ - cout << endl << - ".================================================================." << endl << - //"|| *hardseed* vx.y.z ||" << endl << - "|| " << g_softname << " v" << g_version << " ||" << endl << - "|'--------------------------------------------------------------'|" << endl << - "|| -- SEX IS ZERO (0), so, who wanna be the ONE (1), aha? ||" << endl << - "|'=============================================================='|" << endl << - "|| .::::. ||" << endl << - "|| .::::::::. ||" << endl << - "|| ::::::::::: ||" << endl << - "|| ':::::::::::.. ||" << endl << - "|| .:::::::::::::::' ||" << endl << - "|| '::::::::::::::.` ||" << endl << - "|| .::::::::::::::::.' ||" << endl << - "|| .::::::::::::.. ||" << endl << - "|| .::::::::::::::'' ||" << endl << - "|| .:::. '::::::::'':::: ||" << endl << - "|| .::::::::. ':::::' ':::: ||" << endl << - "|| .::::':::::::. ::::: '::::. ||" << endl << - "|| .:::::' ':::::::::. :::::. ':::. ||" << endl << - "|| .:::::' ':::::::::.::::::. '::. ||" << endl << - "|| .::::'' ':::::::::::::::' '::. ||" << endl << - "|| .::'' '::::::::::::::' ::.. ||" << endl << - "|| ..:::: ':::::::::::' :'''` ||" << endl << - "|| ..''''':' '::::::.' ||" << endl << - "|'=============================================================='|" << endl << - //"|| yangyang.gnu@gmail.com ||" << endl << - "|| " << g_myemail_color << " ||" << endl << - //"|| http://www.yangyangwithgnu.net/ ||" << endl << - "|| " << g_mywebspace_color << " ||" << endl << - "'================================================================'" << endl; -} - -static void -showHelpInfo (void) -{ - cout << endl; - cout << g_softname - << " is a batch seeds and pictures download utiltiy from CaoLiu and AiCheng forum. " - << "It's easy and simple to use. Usually, you could issue it as follow: " << endl - << " $ hardseed" << endl - << "or" << endl - << " $ hardseed --saveas-path ~/downloads --multi-threads 4 --topics-range 8 64" - << " --av-class aicheng_west --timeout-download-picture 32 --hate X-Art --proxy http://127.0.0.1:8087" << endl; - - cout << endl; - cout << " --help" << endl - << " Show this help infomation what you are seeing. " << endl; - - cout << endl; - cout << " --version" << endl - << " Show current version. " << endl; - - cout << endl; - cout << " --av-class" << endl - << " There are 13 av classes: caoliu_west_reposted, caoliu_cartoon_reposted, " - << "caoliu_asia_mosaicked_reposted, caoliu_asia_non_mosaicked_reposted, caoliu_west_original, " - << "caoliu_cartoon_original, caoliu_asia_mosaicked_original, caoliu_asia_non_mosaicked_original, " - << "caoliu_selfie, aicheng_west, aicheng_cartoon, aicheng_asia_mosaicked and aicheng_asia_non_mosaicked. " << endl - << " As the name implies, \"caoliu\" stands for CaoLiu forum, \"aicheng\" for AiCheng forum, " - << "\"reposted\" and \"original\" is clearity, and the \"selfie\" is photos by oneself, you konw " - << "which one is your best lover (yes, only one). " << endl - << " The default is aicheng_asia_mosaicked. " << endl; - - cout << endl; - cout << " --concurrent-tasks" << endl - << " You can set more than one proxy, each proxy could more than one concurrent tasks. This option " - << "set the number of concurrent tasks of each prox. " << endl - << " The max and default number is 8. " << endl; - - cout << endl; - cout << " --timeout-download-picture" << endl - << " Some pictures too big to download in few seconds. So, you should set the download picture " - << "timeout seconds. " << endl - << " The default timeout is 32 seconds." << endl; - - cout << endl; - cout << " --topics-range" << endl - << " Set the range of to download topics. E.G.: " << endl - << " --topics-range 2 16" << endl - << " --topics-range 8 (I.E., --topics-range 1 8)" << endl - << " --topics-range -1 (I.E., all topics of this av class)" << endl - << " The default topics range is 128" << endl; - - cout << endl; - cout << " --saveas-path" << endl - << " Set the path to save seeds and pictures. The rule of dir: [avclass][range]@hhmmss. E.G., " - << "[aicheng_west][2~32]@124908/. " << endl - << " The default directory is home directory (or windows is C:\\). " << endl; - - cout << endl; - cout << " --hate" << endl - << " If you hate some subject topics, you can ignore them by setting this option with keywords " - << "in topic title, split by space-char ' ', and case sensitive. E.G., --hate 孕妇 重口味. " - << "When --hate keywords list conflict with --like, --hate first. " << endl; - - cout << endl; - cout << " --like" << endl - << " If you like some subject topics, you can grab them by setting this option with keywords " - << "in topic title, split by space-char ' ', and case sensitive. E.G., --like 苍井空 小泽玛利亚. " - << "When --like keywords list conflict with --hate, --hate first. " << endl; - - cout << endl; - cout << " --proxy" << endl - << " As you know, the government likes blocking adult websites, so, I do suggest you to set " - << "--proxy option. Hardseed supports more proxys: " << endl - << " a) GoAgent (STRONGLY recommended), --proxy http://127.0.0.1:8087" << endl - << " b) shadowsocks, --proxy socks5://127.0.0.1:1080, or socks5h://127.0.0.1:1080" << endl - << " c) SSH, --proxy socks4://127.0.0.1:7070" << endl - << " d) VPN (PPTP and openVPN), --proxy \"\"" << endl - << " It is important that you should know, you can set more proxys at the same time, split by " - << "space-char ' '. As the --concurrent-tasks option says, each proxy could more than one concurrent " - << "tasks, now, what about more proxys? Yes, yes, the speed of downloading seed and pictures is " - << "very very fast. E.G., --concurrent-tasks 8 --proxy http://127.0.0.1:8087 socks5://127.0.0.1:1080 " - << "socks4://127.0.0.1:7070, the number of concurrent tasks is 8*3. " << endl - << " If you wanna how to install and configure various kinds of proxy, please access my homepage " - << "\"3.3 搭梯翻墙\" http://www.yangyangwithgnu.net/the_new_world_linux/#index_3_3 " << endl - << " The default http://127.0.0.1:8087. " << endl; - - cout << endl; - cout << " That's all. Any suggestions let me know by " - << g_myemail_color - << " or " - << g_mywebspace_color - << ", big thanks to you. Kiddo, take care of your body. :-)" << endl << endl; -} - -static void -showVersionInfo (void) -{ - cout << "hardseed version " << g_version << endl - << "email " << g_myemail << endl - << "webspace " << g_mywebspace << endl << endl; -} - -static bool -parseTopicsRangeArgument ( const vector& topicsrange_arguments_list, - unsigned& topics_range_begin, - unsigned& topics_range_end ) -{ - if (topicsrange_arguments_list.empty()) { - return(false); - } - - string begin_str = topicsrange_arguments_list[0]; - string end_str; - if (topicsrange_arguments_list.size() < 2) { - end_str = begin_str; - begin_str = "1"; - } else { - end_str = topicsrange_arguments_list[1]; - } - - unsigned begin_tmp; - if ( 0 == (begin_tmp = strtoul(begin_str.c_str(), nullptr, 0)) && - '0' != begin_str[0] ) { - return(false); - } - unsigned end_tmp; - if ( 0 == (end_tmp = strtoul(end_str.c_str(), nullptr, 0)) && - '0' != end_str[0] ) { - return(false); - } - - if (begin_tmp > end_tmp) { - return(false); - } - - topics_range_begin = begin_tmp; - topics_range_end = end_tmp; - - - return(true); -} - - -int -main (int argc, char* argv[]) -{ - // parse command line options - // >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> - // >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> - - CmdlineOption cmdline_options((unsigned)argc, argv); - vector cmdline_arguments_list; - - // --help, first high priority, ignore other options - if (cmdline_options.hasOption("--help")) { - showHelpInfo(); - return(EXIT_SUCCESS); - } - - // --version, second high priority, ignore other options - if (cmdline_options.hasOption("--version")) { - showVersionInfo(); - return(EXIT_SUCCESS); - } - - // show the sexy girl ASCII art - showSexyGirl(); - cout << endl; - - // prompt turn on the goagent - cout << RichTxt::bold_on - << "************************ !! IMPORTANCE !! ************************" << endl - << "******** please make sure the proxy software is running ********" << endl - << "************************ !! IMPORTANCE !! ************************" << endl - << RichTxt::bold_off << endl; - - // --av-class - // >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> - cout << "Your command arguments: " << endl; - string av_class_name("aicheng_asia_mosaicked"); - cmdline_arguments_list = cmdline_options.getArgumentsList("--av-class"); - if (!cmdline_arguments_list.empty()) { - av_class_name = cmdline_arguments_list[0]; - } - - bool b_aicheng = true; - Caoliu::AvClass caoliu_av_class = Caoliu::asia_mosaicked_original; - Aicheng::AvClass aicheng_av_class = Aicheng::asia_mosaicked; - if ("caoliu_west_original" == av_class_name) { - caoliu_av_class = Caoliu::west_original; - b_aicheng = false; - } else if ("caoliu_cartoon_original" == av_class_name) { - caoliu_av_class = Caoliu::cartoon_original; - b_aicheng = false; - } else if ("caoliu_asia_mosaicked_original" == av_class_name) { - caoliu_av_class = Caoliu::asia_mosaicked_original; - b_aicheng = false; - } else if ("caoliu_asia_non_mosaicked_original" == av_class_name) { - caoliu_av_class = Caoliu::asia_non_mosaicked_original; - b_aicheng = false; - } else if ("caoliu_west_reposted" == av_class_name) { - caoliu_av_class = Caoliu::west_reposted; - b_aicheng = false; - } else if ("caoliu_cartoon_reposted" == av_class_name) { - caoliu_av_class = Caoliu::cartoon_reposted; - b_aicheng = false; - } else if ("caoliu_asia_mosaicked_reposted" == av_class_name) { - caoliu_av_class = Caoliu::asia_mosaicked_reposted; - b_aicheng = false; - } else if ("caoliu_asia_non_mosaicked_reposted" == av_class_name) { - caoliu_av_class = Caoliu::asia_non_mosaicked_reposted; - b_aicheng = false; - } else if ("caoliu_selfie" == av_class_name) { - caoliu_av_class = Caoliu::selfie; - b_aicheng = false; - } else if ("aicheng_west" == av_class_name) { - aicheng_av_class = Aicheng::west; - } else if ("aicheng_asia_mosaicked" == av_class_name) { - aicheng_av_class = Aicheng::asia_mosaicked; - } else if ("aicheng_cartoon" == av_class_name) { - aicheng_av_class = Aicheng::cartoon; - } else if ("aicheng_asia_non_mosaicked" == av_class_name) { - aicheng_av_class = Aicheng::asia_non_mosaicked; - } else { - cerr << "ERROR! the --av-class argument invalid. More info please see --help. " << endl; - return(EXIT_FAILURE); - } - - cout << " the av class \"" << RichTxt::bold_on << av_class_name << RichTxt::bold_off << "\"; " << endl; - // <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< - - // --timeout-download-picture - unsigned timeout_download_pic = 32; // default timeout seconds - cmdline_arguments_list = cmdline_options.getArgumentsList("--timeout-download-picture"); - if (!cmdline_arguments_list.empty()) { - unsigned tmp = strtoul(cmdline_arguments_list[0].c_str(), nullptr, 0); - if (tmp > 0) { - timeout_download_pic = tmp; - } - } - cout << " the download picture timeout seconds " - << "\"" << RichTxt::bold_on << timeout_download_pic << RichTxt::bold_off << "\"; " << endl; - - // --topics-range, E.G.: - // --topics-range 2 16, - // --topics-range 8 (I.E., --topics-range 1 8), - // --topics-range 4 -1 (I.E., all topics of this class av) - unsigned topics_range_begin = 1, topics_range_end = 128; // default range - cmdline_arguments_list = cmdline_options.getArgumentsList("--topics-range"); - if (!cmdline_arguments_list.empty()) { - if (!parseTopicsRangeArgument(cmdline_arguments_list, topics_range_begin, topics_range_end)) { - cerr << "ERROR! --topics-range argument setting wrong! " << endl; - return(EXIT_FAILURE); - } - } - cout << " the range of parsing topics " - << RichTxt::bold_on << "[" << topics_range_begin << "~" << topics_range_end << "]" << RichTxt::bold_off - << "; " << endl; - - // --saveas-path. - // create dir to save seeds and pictures. the rule of dir: [avclass][range]@hhmmss. - // E.G., [aicheng_west][8~16]@211159. default home directory. - // >>>>>>>>>>>>>>>>>> - string path; - - cmdline_arguments_list = cmdline_options.getArgumentsList("--saveas-path"); - if (cmdline_arguments_list.empty()) { -#ifdef CYGWIN - const char* p_home = "C:\\"; -#else - const char* p_home = getenv("HOME"); -#endif - if (nullptr == p_home) { - cerr << "ERROR! --saveas-path argument setting wrong! " << endl; - return(EXIT_FAILURE); - } - path = p_home; - } else { - path = cmdline_arguments_list[0]; - } - path += "/["; - - // 1st, av class - path += av_class_name; - path += "]"; - - // 2nd, range - path += "[" + convNumToStr(topics_range_begin) + "~" + convNumToStr(topics_range_end) + "]@"; - - // 3rd, time - Time current_time; - path += current_time.getHour(2) + current_time.getMinute(2) + current_time.getSecond(2); - path += "/"; - -#ifdef CYGWIN - // windows path style - replace(path.begin(), path.end(), '/', '\\'); -#endif - - // create dir - if (-1 == mkdir(path.c_str(), 0755)) { - cerr << "ERROR! cannot create " << path << ", " << strerror(errno) << endl; - return(EXIT_FAILURE); - } - -#ifndef CYGWIN - // convert raw path to standard absolute path. To call realpath() success, - // path must have created. - char buffer[PATH_MAX]; - realpath(path.c_str(), buffer); - path = buffer; -#endif - - cout << " the path to save seeds and pictures \"" << RichTxt::bold_on << path << RichTxt::bold_off << "\"; " << endl; - // <<<<<<<<<<<<<<<<<< - - // --concurrent-tasks - unsigned threads_total = 8; // the default and the max number of threads - cmdline_arguments_list = cmdline_options.getArgumentsList("--concurrent-tasks"); - if (!cmdline_arguments_list.empty()) { - unsigned tmp = strtoul(cmdline_arguments_list[0].c_str(), nullptr, 0); - if (tmp <= 0 || tmp > threads_total) { - cerr << "ERROR! --concurrent-tasks argument setting wrong. The max and default number is 16. " << endl; - return(EXIT_FAILURE); - } - threads_total = tmp; - } - cout << " the number of concurrent tasks \"" << RichTxt::bold_on << threads_total << RichTxt::bold_off << "\"; " << endl; - - // --hate. ignore the topics by user setting keywords in topic title, split by space-char ' '. - // for example: --ignore aa bb cc "d d". - vector hate_keywords_list = { "连发", "連发", "连發", "連發", - "è¿žå¼¹", "★㊣", "合辑", "合集", - "合輯", "nike", "最新の美女骑兵㊣", - "精選", "精选" }; // force to ignore the all-in-one topics - cmdline_arguments_list = cmdline_options.getArgumentsList("--hate"); - if (!cmdline_arguments_list.empty()) { - hate_keywords_list.insert(hate_keywords_list.end(), cmdline_arguments_list.begin(), cmdline_arguments_list.end()); - } - cout << " ignore some topics which include the keywords as follow \"" << RichTxt::bold_on; - if (hate_keywords_list.empty()) { - cout << " "; - } else { - copy(hate_keywords_list.cbegin(), hate_keywords_list.cend(), ostream_iterator(cout, ", ")); - } - cout << RichTxt::bold_off << "\b\b\"; " << endl; - - // --like. - vector like_keywords_list; - cmdline_arguments_list = cmdline_options.getArgumentsList("--like"); - if (!cmdline_arguments_list.empty()) { - like_keywords_list.assign(cmdline_arguments_list.begin(), cmdline_arguments_list.end()); - } - cout << " just parse topics which include the kewords as follow \"" << RichTxt::bold_on; - if (like_keywords_list.empty()) { - cout << " "; - } else { - copy(like_keywords_list.cbegin(), like_keywords_list.cend(), ostream_iterator(cout, ", ")); - } - cout << RichTxt::bold_off << "\b\b\"; " << endl; - - // --proxy. prompt user to use proxy, because the caoliu bbs maybe block IP - vector proxy_addrs_list = {"http://127.0.0.1:8087"}; // the default proxy is GoAgent - cmdline_arguments_list = cmdline_options.getArgumentsList("--proxy"); - if (!cmdline_arguments_list.empty()) { - proxy_addrs_list = cmdline_arguments_list; - } - cout << " the proxy \"" << RichTxt::bold_on; - copy(proxy_addrs_list.cbegin(), proxy_addrs_list.cend(), ostream_iterator(cout, ", ")); - cout << "\b\b" << RichTxt::bold_off << "\". " << endl << endl; - - // <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< - // <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< - - - // download pictures and seed - if (b_aicheng) { - Aicheng aicheng( aicheng_av_class, - proxy_addrs_list, - topics_range_begin, topics_range_end, - hate_keywords_list, - like_keywords_list, - threads_total * proxy_addrs_list.size(), - timeout_download_pic, - path ); - } else { - Caoliu caoliu ( caoliu_av_class, - proxy_addrs_list, - topics_range_begin, topics_range_end, - hate_keywords_list, - like_keywords_list, - threads_total * proxy_addrs_list.size(), - timeout_download_pic, - path ); - } - - - cout << endl; - return(EXIT_SUCCESS); -} -