whisker.py

# whisker.py

# 7 Feb 2010

# -----------------------------------------------------------

# --------------------------- Networking routines.

# -----------------------------------------------------------

import re

def connect_both_ports(server, mainport):

   """Connect the main and immediate ports to the server."""

   connect_main(server, mainport) # Log in to the server.

   # Listen to the server until we can connect the immediate socket.

   for line in getlines_mainsock():

       # The server has sent us a message via the main socket.

       if verbosenetwork:

           print "SERVER: " + line # Print the message, for info only.

       m = re.search(r"^ImmPort: (\d+)", line)

       if m:

           immport = m.group(1)

       m = re.search(r"^Code: (\w+)", line)

       if m:

           code = m.group(1)

           connect_immediate(server, immport, code)

           break

import socket

def connect_main(server, portstring):

   """Connect the main port to the server."""

   print "Connecting main port to server."

   m = re.match(r"\D", portstring) # search for \D = non-digit characters

   if m:

       port = socket.getservbyname(portstring, "tcp")

   else:

       port = int(portstring)

   proto = socket.getprotobyname("tcp")

   global mainsock

   try:

       mainsock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, proto)

       mainsock.connect( (server, port) )

   except socket.error, msg:

       mainsock.close()

       mainsock = None

       print "ERROR creating/connecting main socket: "+msg

   print "Connected to main port " + str(port) + " on server " + server

   mainsock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) # Disable the Nagle algorithm.

import time

def connect_immediate(server, portstring, code):

   m = re.match(r"\D", portstring) # search for \D = non-digit characters

   if m:

       port = socket.getservbyname(portstring, "tcp")

   else:

       port = int(portstring)

   proto = socket.getprotobyname("tcp")

   global immsock

   try:

       immsock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, proto)

       immsock.connect( (server, port) )

   except socket.error, msg:

       immsock.close()

       immsock = None

       print "ERROR creating/connecting immediate socket: "+msg

   print "Connected to immediate port " + str(port) + " on server " + server

   immsock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) # Disable the Nagle algorithm.

   immsock.setblocking(True)

   send_immediate("Link "+code)

   sleeptime = 0.1

   print "Sleeping for "+str(sleeptime)+" seconds as the Nagle-disabling feature of Python isn't working properly..."

   time.sleep(sleeptime) # the Nagle business isn't working; the Link packet is getting amalgamated with anything the main calling program starts to send. So pause.

   print "... continuing. Immediate socket should now be correctly linked."

def log_out():

   try:

       mainsock.close()

   except socket.error, msg:

       print "Error closing main socket: "+msg

   try:

       immsock.close()

   except socket.error, msg:

       print "Error closing immediate socket: "+msg

def send(s):

   # Send something to the server on the main socket, with a trailing newline.

   if verbosenetwork:

       print "Main socket command: " + s # For info only.

   mainsock.send(s + "\n")

def send_immediate(s):

   # Send a command to the server on the immediate socket, and retrieve its reply.

   if verbosenetwork:

       print "Immediate socket command: " + s # For info only.

   immsock.sendall(s + "\n")

   reply = getlines_immsock().next()

   if verbosenetwork:

       print "Immediate socket reply:", reply # For info only.

   return reply

# http://stackoverflow.com/questions/822001/python-sockets-buffering

def getlines_immsock():

   # Yield a set of lines from the socket

   buffer = immsock.recv(4096)

   done = False

   while not done:

       if "\n" in buffer:

           (line, buffer) = buffer.split("\n", 1)

           yield line

       else:

           more = immsock.recv(4096)

           if not more:

               done = True

           else:

               buffer = buffer+more

   if buffer:

       yield buffer

def getlines_mainsock():

   # Yield a set of lines from the socket

   buffer = mainsock.recv(4096)

   done = False

   while not done:

       if "\n" in buffer:

           (line, buffer) = buffer.split("\n", 1)

           yield line

       else:

           more = mainsock.recv(4096)

           if not more:

               done = True

           else:

               buffer = buffer+more

   if buffer:

       yield buffer

# -----------------------------------------------------------

# --------------------------- User interface routines.

# -----------------------------------------------------------

def ask_user(prompt, default):

   """Prompts the user, with a default. Returns a string."""

   result = raw_input(prompt + " [" + default + "]: ")

   return result if len(result)>0 else default

def ask_user_password(prompt):

   """Read a password from the console."""

   import getpass

   return getpass.getpass(prompt + ": ")

# -----------------------------------------------------------

# --------------------------- Database routines, using ODBC.

# -----------------------------------------------------------

# Open database connection.

# There's no direct Python equivalent of DBI (which can talk to e.g. ODBC and MySQL).

# So we'll use ODBC. On Windows, that comes by default.

# For IODBC: sudo apt-get install iodbc libiodbc2-dev. Then see http://www.iodbc.org/dataspace/iodbc/wiki/iODBC/IODBCPythonHOWTO . This uses ~/.odbc.ini .

# Or, for MySQL, sudo apt-get install iodbc libmyodbc. Then see https://help.ubuntu.com/community/ODBC . This uses /etc/odbc.ini .

# Or UnixODBC: sudo apt-get install unixodbc unixodbc-dev. Then see http://ubuntu-virginia.ubuntuforums.org/showthread.php?p=5846508 . WE'RE GOING THIS WAY.

# The Python interface to ODBC is pyodbc: http://code.google.com/p/pyodbc/wiki/GettingStarted

# To install pyodbc (see in part http://www.easysoft.com/developer/languages/python/pyodbc.html):

#   * sudo apt-get install python-all-dev (to get development headers)

#   * download e.g. pyodbc-2.1.6.zip from http://code.google.com/p/pyodbc/downloads/list

#   * unzip pyodbc-2.1.6.zip

#   * cd pyodbc-2.1.6

#   * amend setup.py: FOR IODBC: change "libraries.append('odbc')" to "libraries.append('iodbc')"...

#   * amend setup.py: FOR LIBMYODBC: not yet worked out

#   * amend setup.py: FOR UNIXODBC: works as is

#   * sudo python setup.py install

# Now, for unixodbc, set it up:

#   * edit /etc/odbcinst.ini to be e.g.:

#       [myodbc]

#       Description = MySQL ODBC 3.51 Driver (this can be an arbitrary name)

#       Driver = /usr/lib/odbc/libmyodbc.so

#       Setup = /usr/lib/odbc/libodbcmyS.so

#       FileUsage       = 1

#   * edit /etc/odbc.ini to be e.g.

#       [mysql-testdb]

#       Driver       = myodbc

#       Description  = mysql_egret_testdb NEEDS SSH TUNNEL

#       SERVER       = 127.0.0.1 # do not use "localhost" or the driver will look in /var/run/mysqld/mysqld.sock, instead of looking at PORT

#       PORT         = 3306

#       Database     = testdb

#       OPTION       = 3

# Now test:

#   * isql mysql-testdb USER PASSWORD

#   * python tests/dbapitests.py python tests/dbapitests.py "DSN=mysql-testdb;UID=xxx;PWD=xxx"

import pyodbc

def connect_to_database():

   print "--- Enter database details."

   dsn = ask_user("ODBC data source name (DSN)", "mysql-testdb")

   username = ask_user("Database username", "root")

   password = ask_user_password("Database password")

   print "Connecting to database."

   dbh = pyodbc.connect("DSN={0};UID={1};PWD={2}".format(dsn, username, password))

   # XXX print error message if it fails

   # XXX print success message it it succeeds

   # ... at present, it crashes with the error message.

   return dbh

def sql_insert_record(dbh, table, fields, values):

   if len(fields) != len(values):

       print "Field/value mismatch to sql_insert_record()"

       return

   # SQL e.g. INSERT INTO table (field1, field2, field3) VALUES (value1, value2, value3);

   # but we start with INSERT INTO table (field1, field2, field3) VALUES (?, ?, ?);

   sql = "INSERT INTO " + table + " ("

   for i in range(len(fields)):

       if i>0:

           sql += ", "

       sql += fields[i]

   sql += ") VALUES ("

   for i in range(len(values)):

       if i>0:

           sql += ", "

       sql += "?"

   sql += ");"

   if verbosedatabase:

       print "SQL so far (? will be bound to values later): " + sql

   cursor = dbh.cursor()

   cursor.execute(sql, values) # binds the ? to values in the process

   dbh.commit()

def sql_insert_multiple_records(dbh, table, fields, records):

   for record in records:

       sql_insert_record(dbh, table, fields, record)

# -----------------------------------------------------------

# --------------------------- Textfile results storage.

# -----------------------------------------------------------

# Trivial, but...

def produce_csv_output(filehandle, fields, values):

   """Produce CSV output, without using csv.writer, so the log can be used for lots of things."""

   output_csv(filehandle, fields)

   for row in values:

       output_csv(filehandle, row)

def output_csv(filehandle, values):

   line = ""

   for i in range(len(values)):

       if i>0:

           line += ","

       line += values[i]

   filehandle.write(line+"\n")