I recently forked twisted-socks to add SOCKS 5 support for my GoogleScraper in order to scraper Google pages asynchronously. Obviously I needed SOCKS5 support to anonymize the parallel requests such that I can scrape more pages simultaneously.
I tested the code for SOCKS4 and SOCKS4a with a local TOR proxy and
twistd -n socks and the SOCKS5 protocol with the dante socks proxy
server on my VPS. So I guess the
basic functionality should be working by now. GSSAPI (Kerberos)
support is planned.
Here is the socksclient code, which is also available on my github repository:
# Copyright (c) 2011-2013, The Tor Project # See LICENSE for the license. # Updated on 25.01.14-28.01.14 to add SOCKS 5 support. # Cleaned some parts of the code and abstracted quite a bit to handle the most important SOCKS5 # functionality like # - username/password authentication # - gssapi authentication (planned) # - CONNECT command (the normal case, there are others: UDP ASSOCIATE and BIND, but they aren't as important. Maybe I will add them # in the future. If anyone wants to implement them, the basic structure is already here and the SOCKSv5ClientProtocol should be # rather easy extensible (how the actual connection, listening for incoming connections (BIND) and opening a UDP connection (UDP ASSOCIATE) # is done in the twisted world, is another question. # Added: # - SOCKSv4ClientFactory was renamed to SOCKSClientFactory and abstracted to handle all SOCKS 4/4a SOCKS5 (It is still ONE protocol, so one Factory should be logical correct) # - added SOCKS5ClientFactory # - SOCKSClientProtocol is the base class for all three protocols # - SOCKSv4aClientProtocol inherits from SOCKSv4ClientProtocol. I made the deliberate choice to differ between SOCKS 4 and 4a, altough 4a has the exactly same functionality as 4, # it might be the case that servers only speak version 4. # References: # A actively maintained, most recent version of PySocks from https://github.com/Anorov/PySocks # The original version of socksclient.py: # Author: Nikolai Tschacher # Contact: incolumitas.com import inspect import socket import re import struct from zope.interface import implements from twisted.internet import defer from twisted.internet.interfaces import IStreamClientEndpoint, IReactorTime from twisted.internet.protocol import Protocol, ClientFactory from twisted.internet.endpoints import _WrappingFactory class SOCKSError(Exception): def __init__(self, val): self.val = val def __str__(self): return repr(self.val) class SOCKSClientProtocol(Protocol): ''' Base class for SOCKS protocols 4, 4a and 5 ''' buf = '' def noteTime(self, event): if self._timer: self._timestamps[event] = self._timer.seconds() def abort(self, errmsg): self.transport.loseConnection() self.handshakeDone.errback(SOCKSError('SOCKS %s: %s' % (self.proxy_config['version'], errmsg))) def isHostname(self, string): dns_label_regex = re.compile(r'^(?![0-9]+$)(?!-)[a-zA-Z0-9-]{,63}(?H", port) self.transport.write(msg) self.noteTime('RELAY_REQUEST_SENT') self.protocol_state = 'connection_requested' def verifySocksReply(self, data): where = 'SOCKS5 verifySocksReply' if len(data) < 10: # all hostname are longer than a IPv4 address self.abort('Too few data from server %s.' % where) else: version, reply, rsv, address_type = struct.unpack('!BBBB', data[:4]) if version != 0x5: self.abort('Invalid version') return False if reply != 0x0: self.abort('Server reply indicates failure. Reason: %s' % self.SOCKS5_ERRORS.get(reply, "Unknown error")) return False if address_type == 0x1: # handle IPv4 address self.bound_address, self.bound_port = socket.inet_ntoa(data[4:8]), struct.unpack('>H', data[8:10])[0] elif address_type == 0x3: # handle domain name dns_name_len = ord(data[4:5]) self.bound_address, self.bound_port = data[5:dns_name_len], struct.unpack('>H', data[(5+dns_name_len):(5+dns_name_len+2)])[0] elif address_type == 0x4: # handle Ipv6 address self.bound_address, self.bound_port = socket.inet_ntop(socket.AF_INET6, data[4:20]), struct.unpack('>H', data[20:22])[0] self.protocol_state = 'connection_verified' return True def connectionMade(self): self.noteTime('CONNECTED') self.noteTime('NEGOTIATE_AUTH_METHOD') self.negotiateAuthenticationMethod() def dataReceived(self, data): self.buf += data if self.protocol_state == 'do_auth': self.authenticate(data) elif self.protocol_state == 'check_auth': self.checkAuth(data) if self.protocol_state == 'authenticated': host = self.postHandshakeEndpoint._host port = self.postHandshakeEndpoint._port self.sendRelayRequest(host, port) elif self.protocol_state == 'connection_requested': if self.verifySocksReply(data): self.setupRelay() class SOCKSv4ClientProtocol(SOCKSClientProtocol): SOCKS4_ERRORS = { 0x5B: "Request rejected or failed", 0x5C: "Request rejected because SOCKS server cannot connect to identd on the client", 0x5D: "Request rejected because the client program and identd report different user-ids" } def sendRelayRequest(self, host, port): username = self.proxy_config['version_specific']['username'] ver, cmd, username = 0x4, 0x1, [b'\x00', username.encode()+b'\x00'][not not username] try: addr = socket.inet_aton(host) except socket.error: self.abort('Not a valid IPv4 address.') return False msg = struct.pack('!BBH', ver, cmd, port) + addr + username self.transport.write(msg) self.noteTime('REQUEST') def verifySocksReply(self, data): """ Return True on success and False on need-more-data or error. In the case of an error, the connection is closed and the handshakeDone errback is invoked with a SOCKSError exception before False is returned. """ if len(data) < 8: return False if ord(data[0]) != 0x0: self.abort('Expected 0 bytes') return False status = ord(data[1]) if status != 0x5a: self.abort('Relay request failed. Reason=%s.' % self.SOCKS4_ERRORS.get(data[0], 'Unknown error')) return False return True def connectionMade(self): self.noteTime('CONNECT') self.noteTime('NEGOTIATE') self.sendRelayRequest(self.postHandshakeEndpoint._host, self.postHandshakeEndpoint._port) def dataReceived(self, data): self.buf += data if self.verifySocksReply(data): self.setupRelay() class SOCKSv4aClientProtocol(SOCKSv4ClientProtocol): '''Only extends SOCKS 4 to remotely resolve hostnames.''' def sendRelayRequest(self, host, port): username = self.proxy_config['version_specific']['username'] ver, cmd, username = 0x4, 0x1, [b'\x00', username.encode()+b'\x00'][not not username] try: addr = socket.inet_aton(host) except socket.error: addr = '\x00\x00\x00\x01' dnsname = '%s\x00' % host msg = struct.pack('!BBH', ver, cmd, port) + addr + username + dnsname else: msg = struct.pack('!BBH', ver, cmd, port) + addr + username self.transport.write(msg) self.noteTime('REQUEST') class SOCKSClientFactory(ClientFactory): def __init__(self, proxy_config): self.proxy_config = proxy_config if self.proxy_config['version'] == '4': self.protocol = SOCKSv4ClientProtocol elif self.proxy_config['version'] == '4a': self.protocol = SOCKSv4aClientProtocol elif self.proxy_config['version'] == '5': self.protocol = SOCKSv5ClientProtocol def buildProtocol(self, addr): r = ClientFactory.buildProtocol(self, addr) r.proxy_config = self.proxy_config r.postHandshakeEndpoint = self.postHandshakeEndpoint r.postHandshakeFactory = self.postHandshakeFactory r.handshakeDone = self.handshakeDone r._timestamps = self._timestamps r._timer = self._timer return r class SOCKSWrapper(object): ''' Generic class to wrap all 3 SOCKS protocol versions 4, 4a, 5 around a TCP connection ''' implements(IStreamClientEndpoint) factory = SOCKSClientFactory def __init__(self, reactor, endpoint, proxy_config, timestamps=None): self._host = proxy_config['host'] self._port = proxy_config['port'] self._proxy_config = proxy_config self._reactor = reactor self._endpoint = endpoint self._timestamps = None self._timer = None if timestamps is not None: self._timestamps = timestamps self._timer = IReactorTime(reactor) def noteTime(self, event): if self._timer: self._timestamps[event] = self._timer.seconds() def connect(self, protocolFactory): """ Return a deferred firing when the SOCKS connection is established. """ def createWrappingFactory(f): """ Wrap creation of _WrappingFactory since __init__() doesn't take a canceller as of Twisted 12.1 or something. """ if len(inspect.getargspec(_WrappingFactory.__init__)[0]) == 3: def _canceller(deferred): connector.stopConnecting() deferred.errback( error.ConnectingCancelledError( connector.getDestination())) return _WrappingFactory(f, _canceller) else: # Twisted >= 12.1. return _WrappingFactory(f) self.noteTime('START') try: # Connect with an intermediate SOCKS factory/protocol, # which then hands control to the provided protocolFactory # once a SOCKS connection has been established. f = self.factory(self._proxy_config) f.postHandshakeEndpoint = self._endpoint f.postHandshakeFactory = protocolFactory f.handshakeDone = defer.Deferred() f._timestamps = self._timestamps f._timer = self._timer wf = createWrappingFactory(f) self._reactor.connectTCP(self._host, self._port, wf) self.noteTime('SOCKET') return f.handshakeDone except: return defer.fail()
You can use the module for HTTP connection endpoints somehow like that
#!/usr/bin/env python # Copyright (c) 2011-2013, The Tor Project # See LICENSE for the license. import sys from urlparse import urlparse from twisted.internet import reactor, endpoints from socksclient import SOCKSv4ClientProtocol, SOCKSWrapper from twisted.web import client class TestClass: def __init__(self): self.npages = 0 self.timestamps = {} def wrappercb(self, proxy): print "connected to proxy", proxy def clientcb(self, content): print "ok, got: %s" % content[:120] print "timetamps " + repr(self.timestamps) self.npages -= 1 if self.npages == 0: reactor.stop() def sockswrapper(self, proxy_config, url): dest = urlparse(url) assert dest.port is not None, 'Must specify port number.' endpoint = endpoints.TCP4ClientEndpoint(reactor, dest.hostname, dest.port) return SOCKSWrapper(reactor, endpoint, proxy_config, timestamps=self.timestamps) def main(): thing = TestClass() # Mandatory first argument is a URL to fetch over Tor (or whatever # SOCKS proxy that is running on localhost:9050). url = sys.argv[1] proxy_config = { 'host': '127.0.0.1', 'port': 1080, 'version': '4', 'version_specific': { 'rdns': True, # Enforce resolving hostnames remotely (Only supported by version 4a and 5) 'cmd': b'\x01', # this may be CONNECT, BIND and UDP in version 5. In 4 and 4a, it's always CONNECT or BIND 'username': 'socksuser', # Enables simple username/password authentication mechanism in version 5 'password': '' } } proxy_config2 = { 'host': '212.224.92.182', 'port': 7777, 'version': '5', 'version_specific': { 'rdns': True, # Enforce resolving hostnames remotely (Only supported by version 4a and 5) 'cmd': b'\x01', # this may be CONNECT, BIND and UDP in version 5. In 4 and 4a, it's always CONNECT or BIND 'username': 'someuser', # Enables simple username/password authentication mechanism in version 5 'password': 'somepass' } } # From http://fastproxyservers.org/socks5-servers.htm proxy_config3 = { 'host': '202.84.44.129', 'port': 1080, 'version': '4', 'version_specific': { 'rdns': True, # Enforce resolving hostnames remotely (Only supported by version 4a and 5) 'cmd': b'\x01', # this may be CONNECT, BIND and UDP in version 5. In 4 and 4a, it's always CONNECT or BIND 'username': '', # Enables simple username/password authentication mechanism in version 5 'password': '' } } f = client.HTTPClientFactory(url) f.deferred.addCallback(thing.clientcb) sw = thing.sockswrapper(proxy_config2, url) d = sw.connect(f) d.addCallback(thing.wrappercb) thing.npages += 1 reactor.run() if '__main__' == __name__: main()