The Mana World Forums

Code: Select all

import sys
import random
import string

def run(filename=''):
	file = open( filename, 'r')
	
	text = file.read()
	file.close()
	words = string.split(text)
	
	end_sentence = []
	dict = {}
	prev1 = ''
	prev2 = ''
	for word in words:
	  if prev1 != '' and prev2 != '':
	    key = (prev2, prev1)
	    if dict.has_key(key):
	      dict[key].append(word)
	    else:
	      dict[key] = [word]
	      if prev1[-1:] == '.':
	        end_sentence.append(key)
	  prev2 = prev1
	  prev1 = word
	
	if end_sentence == []:
	  answers = ['Sorry, but I don\'t know anything about it!',
	  	     'Are you talking to me?',
		     'Why should I answer?',
		     'Oh, that\'s sad...',
	  	     'Well, I don\'t think I know anything about it']
	  
	  print random.choice(answers)
	  return
	
	key = ()
	count = 2
	
	while 1:
	  if dict.has_key(key):
	    word = random.choice(dict[key])
	    print word,
	    key = (key[1], word)
	    if key in end_sentence:
	      print
	      count = count - 1
	      if count <= 0:
		break
	  else:
	    key = random.choice(end_sentence)
	
if __name__ == '__main__':
	run(sys.argv[1])

Code: Select all

#!/bin/sh

# run marvelous text browser LYNX in special dump mode
# dump output to temp-file
/usr/bin/lynx \
	-cfg=/dev/null \
	-dump \
	-accept_all_cookies \
	-anonymous \
	-assume_charset=utf-8 \
	-assume_unrec_charset=utf-8 \
	-connect_timeout=5 \
	-cookie_file=lynx_cookies \
	-cookie_save_file=lynx_cookies \
	-display_charset=utf-8 \
	-force_html \
	-nobrowse \
	-nolist \
	-nostatus \
   	-ftp \
	-telnet \
	-rlogin \
	-tagsoup \
	-width=10000 \
	-nomargins \
	-nonumbers\
    "$1" > temp-file

# Step .: Expand common problematic abbreviations
# Step 1: Remove all leading and trailing spaces.
# Step 2: Convert invalid punctuation by adding a space.
# Step 3: Remove extra leading spaces on some punctuation.
# Step 4: Remove extra trailing spaces on some punctuation.
# Step 5: Split lines at end of sentences.
# Step 6: Remove [] text, compact spaces, compact pauses.
# Step 7: Select only sentences resembling legal text.
# Step 8: Elliminate broken brackets.
# Step 9: Homogenize quotes.
# Step A: Remove broken quotes.
# Step B: Remove Google search phrases and code fragments.
# Step C: Remove dates.
# Step D: Remove repeated dots.

cat temp-file | \
  sed -e 's/\([dD]\)r. /\1octor /g;s/\([Pp]\)rof. /\1rofessor /g;s/\([Aa]\)pprox. /\1pproximately /g;s/est. /established /g' | \
  sed -e 's/^[ ]*//g;s/[ ]*$//g' | \
  sed -e 's/\([^A-Z]\)\([.!?:,;)]\)\([A-Z]\)/\1\2 \3/g' | \
  sed -e 's/[ ]*\([.!?:,;)]\)/\1/g' | \
  sed -e 's/([ ]*/)/g' | \
  sed -e 's/\([.!?]\) /\1\
/g' | \
  sed -e 's/\[[^[]*\]//g;s/[ ][ ]*/ /g;s/-[-]*/-/g' | \
  grep -I '^[A-Z].* .*[a-z][a-z][a-z].*[.!?]$' | \
  grep -vE '\([^)]*$|^[^(]*\)' | \
  sed -e 's/``/"/g;s/'"''"'/"/g;s/,,/"/g' | \
  grep -vE '"[^"]*$|^[^"]*"' | \
  sed 's/</{/g;s/>/}/g' | \
  sed -e 's/\([^A-Z]\)\([.!?:,;)][.!?:,;)]*\)\([A-Za-z]\)/\1\2 \3/g' | \
  grep -vE ':.*:|}.*}|{.*{' | \
  grep -viE '^Subject[^a-z]|^Re:|^From:|^To:|^Fwd:|^Temat:|^Wpis:' | \
  grep -viE '2011 Google' | \
  grep -viE 'Search within results ' | \
  grep -vie 'Groups View all web results' | \
  grep -vie 'nav_[a-z]*gif' | \
  grep -vE '([A-Z][a-z]{2} [0-9]{1,2} [0-9]{1,4})|([A-Z][a-z]{2} by)' | \
  grep -vE '([0-9]{1,2}/){2,3}' | \
  grep -vE '[.]{3}@[a-z]*' | \
  grep -ve '[A-Z][a-z]{2} [0-9]{1,2}, [0-9]{1,4}' | \
  sed -e 's/\.\././g' | \
  sed -e 's/\.\././g' | \
  sort -u > output-file

# remove temporary file
rm -f temp-file
# execute Mark V Shaney with sentences database
python shaney.py output-file 
# remove temporary file
rm -f output-file