I started by adding a shutter button to the top of the enclosure. I used a Cherry MX Blue mechanical keyboard switch that I had leftover from a project last year.

The battery and Raspberry Pi just barely fit into the enclosure:

The Raspberry Pi camera module is wedged snugly beneath the camera’s front plate:

In additional to playing the song, I added some functionality that provides a bit of context to the user. Using the pico2wave text-to-speech utility, the camera speaks the tags aloud before playing the song. Additionally, using SoX, the camera plays an initialization tone generated from the color histogram of the image before reading the tags.

Here’s the code that’s currently running on the Raspberry Pi:

from __future__ import unicode_literals

import os
import json
import uuid
import time
from random import choice as rc
from random import sample as rs
import re
import subprocess

import RPi.GPIO as GPIO
import picamera
from clarifai.client import ClarifaiApi
import requests
from PIL import Image

import sys
import threading

import spotify

import genius_token

# SPOTIFY STUFF

# Assuming a spotify_appkey.key in the current dir
session = spotify.Session()

# Process events in the background
loop = spotify.EventLoop(session)
loop.start()

# Connect an audio sink
audio = spotify.AlsaSink(session)

# Events for coordination
logged_in = threading.Event()
logged_out = threading.Event()
end_of_track = threading.Event()

logged_out.set()


def on_connection_state_updated(session):
    if session.connection.state is spotify.ConnectionState.LOGGED_IN:
        logged_in.set()
        logged_out.clear()
    elif session.connection.state is spotify.ConnectionState.LOGGED_OUT:
        logged_in.clear()
        logged_out.set()


def on_end_of_track(self):
    end_of_track.set()

# Register event listeners
session.on(
    spotify.SessionEvent.CONNECTION_STATE_UPDATED, on_connection_state_updated)
session.on(spotify.SessionEvent.END_OF_TRACK, on_end_of_track)

# Assuming a previous login with remember_me=True and a proper logout
# session.relogin()
# session.login(genius_token.spotify_un, genius_token.spotify_pwd, remember_me=True)

# logged_in.wait()

# CAMERA STUFF

# Init Camera
camera = picamera.PiCamera()

# Init GPIO
GPIO.setmode(GPIO.BCM)

# Button Pin
GPIO.setup(18, GPIO.IN, pull_up_down=GPIO.PUD_UP)

IMGPATH = '/home/pi/soundcamera/img/'

clarifai_api = ClarifaiApi()

def chunks(l, n):
    """Yield successive n-sized chunks from l."""
    for i in xrange(0, len(l), n):
        yield l[i:i+n]

def take_photo():
    fn = str(int(time.time()))+'.jpg' # TODO: Change to timestamp hash
    fp = IMGPATH+fn
    camera.capture(fp)
    return fp

def chunks(l, n):
    """Yield successive n-sized chunks from l."""
    for i in xrange(0, len(l), n):
        yield l[i:i+n]

def get_tags(fp):
    fileObj = open(fp)
    result = clarifai_api.tag_images(fileObj)
    resultObj = result['results'][0]
    tags = resultObj['result']['tag']['classes']
    return tags

def genius_search(tags):
    access_token = genius_token.token
    payload = {
        'q': ' '.join(tags),
        'access_token': access_token
    }
    endpt = 'http://api.genius.com/search'
    response = requests.get(endpt, params=payload)
    results = response.json()
    hits = results['response']['hits']
    
    artists_titles = []
    
    for h in hits:
        hit_result = h['result']
        if hit_result['url'].endswith('lyrics'):
            artists_titles.append(
                (hit_result['primary_artist']['name'], hit_result['title'])
            )
    
    return artists_titles

def spotify_search(query):
    endpt = "https://api.spotify.com/v1/search"
    payload = {
        'q': query,
        'type': 'track'
    }
    response = requests.get(endpt, params=payload)
    result = response.json()
    result_zero = result['tracks']['items'][0]
    
    return result_zero['uri']

def main(fn):
    tags = get_tags(fn)
    for tag_chunk in chunks(tags,3):
        artists_titles = genius_search(tag_chunk)
        for artist, title in artists_titles:
            try:
                result_uri = spotify_search(artist+' '+title)
            except IndexError:
                pass
            else:
                print tag_chunk
                byline = "%s by %s" % (title, artist)
                print byline
                to_read = ', '.join(tag_chunk) + ". " + byline
                return to_read, result_uri

def play_uri(track_uri):
    # Play a track
    # audio = spotify.AlsaSink(session)
    session.login(genius_token.spotify_un, genius_token.spotify_pwd, remember_me=True)
    logged_in.wait()
    track = session.get_track(track_uri).load()
    session.player.load(track)
    session.player.play()


def stop_track():
    session.player.play(False)
    session.player.unload()
    session.logout()
    logged_out.wait()
    audio._close()

def talk(msg):
    proc = subprocess.Popen(
        ['bash', '/home/pi/soundcamera/play_text.sh', msg]
    )
    proc.communicate()

def play_tone(freqs):
    freq1, freq2 = freqs
    proc = subprocess.Popen(
        ['play', '-n', 'synth', '0.25', 'saw', "%i-%i" % (freq1, freq2)]
    )
    proc.communicate()

def histo_tone(fp):
    im = Image.open(fp)
    hist = im.histogram()
    vals = map(sum, chunks(hist, 64)) # list of 12 values
    print vals
    map(play_tone, chunks(vals,2))

if __name__ == "__main__":
    input_state = True
    new_state = True
    hold_counter = 0
    while 1:
        input_state = GPIO.input(18)
        if not (input_state and new_state):
            talk("capturing")

            # Hold for 15 seconds to turn off
            while not GPIO.input(18):
                time.sleep(0.1)
                hold_counter += 1
                if hold_counter > 150:
                    os.system('shutdown now -h')
                    sys.exit()

            # Reset hold counter
            hold_counter = 0

            # Else take photo
            try:
                img_fp = take_photo()
                msg, uri = main(img_fp)
                histo_tone(img_fp)
                talk(msg)
                play_uri(uri)
            except:
                print sys.exc_info()

            # Wait for playback to complete or Ctrl+C
            try:
                while not end_of_track.wait(0.1):
                    # If new photo, play new song
                    new_state = GPIO.input(18)
                    if not new_state:
                        stop_track()
                        # time.sleep(2)
                        break
            except KeyboardInterrupt:
                pass

I have not yet chosen the authors I plan to embed in these cameras, or how I plan to present the extracted text. I also have tentative plans for a new iteration of the talking surveillance camera I developed last semester, but more on that will be provided in future posts.

This week, I spent some time on eBay finding a few broken medium- and large-format cameras to use as cases. Here’s what I bought (for $5 to $25 each):

I am current waiting to receive them so that I can start planning the builds. Below is a list of the additional parts that will be required for each camera:

Raspberry Pi 2 ($40)
85.60mm x 56mm x 21mm (or roughly 3.37″ x 2.21″ x 0.83″)

Raspberry Pi Camera Board ($30)
25mm x 20mm x 9mm

Buck Converter ($10)
51 * 26.3 * 14 (L * W * H) (mm)

7.4V LiIon Battery Pack ($90)
22mm (0.9″) x 104mm (4.1″) x 107mm (4.2″)
OR two USB batteries ($40)

Thermal Printer ($25 from China or $50 from U.S.)
~4 1/8″ (105mm) x 2 1/4″ (58mm) for rectangular hole
~58mm deep

On/Off Switch ($1)
18.60mm x 12.40mm rectangular hole
13.9mm deep

LED Button ($5)
Shutter button, user will hold for 3 seconds to turn off Raspberry Pi
16mm round hole
~1.5″ deep

1/4-size permaproto board ($3)

1/4″ Acrylic ($12) or Broken Medium Format TLR ($30-69)

Jumper Wires ($2)

For my final projects in Conversation and Computation with Lauren McCarthy and This Is The Remix with Roopa Vasudevan, I iterated on my word.camera project. I added a few new features to the web application, including a private API that I used to enable the creation of a physical version of word.camera inside a Mamiya C33 TLR.

The current version of the code remains open source and available on GitHub, and the project continues to receive positive mentions in the press.

On April 19, I announced two new features for word.camera via the TinyLetter email newsletter I advertised on the site.

Hello,

Thank you for subscribing to this newsletter, wherein I will provide occasional updates regarding my project, word.camera.

I wanted to let you know about two new features I added to the site in the past week:

word.camera/albums You can now generate ebooks (DRM-free ePub format) from sets of lexographs.

word.camera/postcards You can support word.camera by sending a lexograph as a postcard, anywhere in the world for $5. I am currently a graduate student, and proceeds will help cover the cost of maintaining this web application as a free, open source project.

Also:

word.camera/a/XwP59n1zR A lexograph album containing some of the best results I’ve gotten so far with the camera on my phone.

1, 2, 3 A few random lexographs I did not make that were popular on social media.

Best,

Ross Goodwin
rossgoodwin.com
word.camera

Next, I set to work on the physical version. I decided to use a technique I developed on another project earlier in the semester to create word.camera epitaphs composed of highly relevant paragraphs from novels. To ensure fair use of copyrighted materials, I determined that all of this additional data would be processed locally on the physical camera.

I developed a collection of data from a combination of novels that are considered classics and those I personally enjoyed, and I included only paragraphs over 99 characters in length. In total, the collection contains 7,113,809 words from 48 books.

Below is an infographic showing all the books used in my corpus, and their relative included word counts (click on it for the full-size image).

To build the physical version of word.camera, I purchased the following materials:

• Raspberry Pi 2 board
• Raspberry Pi camera module
• Two (2) 10,000 mAh batteries
• Thermal receipt printer
• 40 female-to-male jumper wires
• Three (3) extra-small prototyping perf boards
• LED button

After some tinkering, I was able to put together the arrangement pictured below, which could print raw word.camera output on the receipt printer.

I thought for a long time about the type of case I wanted to put the camera in. My original idea was a photobooth, but I felt that a portable camera—along the lines of Matt Richardson’s Descriptive Camera—might take better advantage of the Raspberry Pi’s small footprint.

Rather than fabricating my own case, I determined that an antique film camera might provide a familiar exterior to draw in people not familiar with the project. (And I was creating it for a remix-themed class, after all.) So I purchased a lot of three broken TLR film cameras on eBay, and the Mamiya C33 was in the best condition of all of them, so I gutted it. (N.B. I’m an antique camera enthusiast—I own a working version of the C33’s predecessor, the C2—and, despite its broken condition, cutting open the bellows of the C33 felt sacrilegious.)

I laser cut some clear acrylic I had left over from the traveler’s lamp project to fill the lens holes and mount the LED button on the back of the camera. Here are some photos of the finished product:

And here is the code that’s running on the Raspberry Pi (the crux of the matching algorithm is on line 90):

import uuid
import picamera
import RPi.GPIO as GPIO
import requests
from time import sleep
import os
import json
from Adafruit_Thermal import *
from alchemykey import apikey
import time

# SHUTTER COUNT / startNo GLOBAL
startNo = 0

# Init Printer
printer = Adafruit_Thermal("/dev/ttyAMA0", 19200, timeout=5)
printer.setSize('S')
printer.justify('L')
printer.setLineHeight(36)

# Init Camera
camera = picamera.PiCamera()

# Init GPIO
GPIO.setmode(GPIO.BCM)

# Working Dir
cwd = '/home/pi/tlr'

# Init Button Pin
GPIO.setup(21, GPIO.IN, pull_up_down=GPIO.PUD_UP)

# Init LED Pin
GPIO.setup(20, GPIO.OUT)

# Init Flash Pin
GPIO.setup(16, GPIO.OUT)

# LED and Flash Off
GPIO.output(20, False)
GPIO.output(16, False)

# Load lit list
lit = json.load( open(cwd+'/lit.json', 'r') )


def blink(n):
    for _ in range(n):
        GPIO.output(20, True)
        sleep(0.2)
        GPIO.output(20, False)
        sleep(0.2)

def takePhoto():
    fn = str(int(time.time()))+'.jpg' # TODO: Change to timestamp hash
    fp = cwd+'/img/'+fn
    GPIO.output(16, True)
    camera.capture(fp)
    GPIO.output(16, False)
    return fp

def getText(imgPath):
    endPt = 'https://word.camera/img'
    payload = {'Script': 'Yes'}
    files = {'file': open(imgPath, 'rb')}
    response = requests.post(endPt, data=payload, files=files)
    return response.text

def alchemy(text):
    endpt = "http://access.alchemyapi.com/calls/text/TextGetRankedConcepts"
    payload = {"apikey": apikey,
               "text": text,
               "outputMode": "json",
               "showSourceText": 0,
               "knowledgeGraph": 1,
               "maxRetrieve": 500}
    headers = {'content-type': 'application/x-www-form-urlencoded'}
    r = requests.post(endpt, data=payload, headers=headers)
    return r.json()

def findIntersection(testDict):
    returnText = ""
    returnTitle = ""
    returnAuthor = ""
    recordInter = set(testDict.keys())
    relRecord = 0.0
    for doc in lit:
        inter = set(doc['concepts'].keys()) & set(testDict.keys())
        if inter:
            relSum = sum([doc['concepts'][tag]+testDict[tag] for tag in inter])
            if relSum > relRecord: 
                relRecord = relSum
                recordInter = inter
                returnText = doc['text']
                returnTitle = doc['title']
                returnAuthor = doc['author']
    doc = {
        'text': returnText,
        'title': returnTitle,
        'author': returnAuthor,
        'inter': recordInter,
        'record': relRecord
    }
    return doc

def puncReplace(text):
    replaceDict = {
        '—': '---',
        '–': '--',
        '‘': "\'",
        '’': "\'",
        '“': '\"',
        '”': '\"',
        '´': "\'",
        'ë': 'e',
        'ñ': 'n'
    }

    for key in replaceDict:
        text = text.replace(key, replaceDict[key])

    return text


blink(5)
while 1:
    input_state = GPIO.input(21)
    if not input_state:
        GPIO.output(20, True)
        try:
            # Get Word.Camera Output
            print "GETTING TEXT FROM WORD.CAMERA..."
            wcText = getText(takePhoto())
            blink(3)
            GPIO.output(20, True)
            print "...GOT TEXT"

            # Print
            # print "PRINTING PRIMARY"
            # startNo += 1
            # printer.println("No. %i\n\n\n%s" % (startNo, wcText))

            # Get Alchemy Data
            print "GETTING ALCHEMY DATA..."
            data = alchemy(wcText)
            tagRelDict = {concept['text']:float(concept['relevance']) for concept in data['concepts']}
            blink(3)
            GPIO.output(20, True)
            print "...GOT DATA"

            # Make Match
            print "FINDING MATCH..."
            interDoc = findIntersection(tagRelDict)
            print interDoc
            interText = puncReplace(interDoc['text'].encode('ascii', 'xmlcharrefreplace'))
            interTitle = puncReplace(interDoc['title'].encode('ascii', 'xmlcharrefreplace'))
            interAuthor = puncReplace(interDoc['author'].encode('ascii', 'xmlcharrefreplace'))
            blink(3)
            GPIO.output(20, True)
            print "...FOUND"

            grafList = [p for p in wcText.split('\n') if p]

            # Choose primary paragraph
            primaryText = min(grafList, key=lambda x: x.count('#'))
            url = 'word.camera/i/' + grafList[-1].strip().replace('#', '')

            # Print
            print "PRINTING..."
            startNo += 1
            printStr = "No. %i\n\n\n%s\n\n%s\n\n\n\nEPITAPH\n\n%s\n\nFrom %s by %s" % (startNo, primaryText, url, interText, interTitle, interAuthor)
            printer.println(printStr)

        except:
            print "SOMETHING BROKE"
            blink(15)

        GPIO.output(20, False)

Thanks to a transistor pulsing circuit that keeps the printer’s battery awake, and some code that automatically tethers the Raspberry Pi to my iPhone, the Fiction Camera is fully portable. I’ve been walking around Brooklyn and Manhattan over the past week making lexographs—the device is definitely a conversation starter. As a street photographer, I’ve noticed that people seem to be more comfortable having their photograph taken with it than with a standard camera, possibly because the visual image (and whether they look alright in it) is far less important.

As a result of these wanderings, I’ve accrued quite a large number of lexograph receipts. Earlier iterations of the receipt design contained longer versions of the word.camera output. Eventually, I settled on a version that contains a number (indicating how many lexographs have been taken since the device was last turned on), one paragraph of word.camera output, a URL to the word.camera page containing the photo + complete output, and a single high-relevance paragraph from a novel.

I also demonstrated the camera at ConvoHack, our final presentation event for Conversation and Computation, which took place at Babycastles gallery, and passed out over 50 lexograph receipts that evening alone.

Photographs by Karam Byun

Often, when photographing a person, the camera will output a passage from a novel featuring a character description that subjects seem to relate to. Many people have told me the results have qualities that remind them of horoscopes.

Here are some additional sketches and photos of the finished product:

The drawer left ample space for a variable voltage power supply (for the solenoids), a pair of speakers (to amplify the music), and my MacBook Pro (to run Stockfish).

Here’s a look beneath the board:

On Wednesday, I presented my progress thus far on the Stenogloves for my final project in Introduction to Physical Computing with Tom Igoe. Since my last post, I have connected the prototype keyboard to an Arduino Micro, developed an algorithm for translating chords into keystrokes, updated the typing tutor game I had demonstrated previously, and iterated through three chord layouts.

Here is the current prototype in action, with my final chord layout and updated typing tutor game:

After connecting the keyboard I discussed in my previous post to an Arduino Micro, I developed the following Arduino sketch for detecting chords and translating them into keystrokes:

int pins[10] = {2, 3, 4, 5, 6, 7, 8, 9, 10, 11};
int keyStatus[10];
int keyStatus2[10];
boolean waiting = false;
char ctrlKey = KEY_LEFT_CTRL;

boolean alt = false;

//int chords[1024] = {0, 116, 115, 117, 114, 0, 0, 118, 111, 39, 62, 0, 112, 0, 113, 119, 32, 46, 58, 93, 59, 0, 0, 125, 44, 0, 9, 0, 91, 0, 123, 45, 0, 84, 83, 85, 82, 0, 0, 86, 79, 0, 0, 0, 80, 0, 81, 87, 10, 0, 0, 0, 0, 0, 0, 0, 63, 47, 0, 0, 0, 0, 0, 0, 110, 0, 0, 0, 0, 0, 0, 0, 120, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 64, 92, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 105, 0, 0, 0, 0, 0, 0, 0, 121, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 107, 0, 0, 0, 108, 0, 109, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 0, 76, 0, 77, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 0, 0, 101, 0, 0, 0, 0, 0, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 0, 0, 0, 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 0, 0, 0, 0, 0, 0, 0, 104, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 126, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500};
//int chords[1024] = {0, 116, 115, 117, 114, 0, 0, 118, 111, 39, 62, 0, 112, 0, 113, 119, 32, 46, 58, 93, 59, 0, 0, 125, 44, 0, 9, 0, 91, 0, 123, 45, 0, 84, 83, 85, 82, 0, 0, 86, 79, 0, 0, 0, 80, 0, 81, 87, 10, 0, 0, 0, 0, 0, 0, 0, 63, 47, 0, 0, 0, 0, 0, 0, 110, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 78, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 92, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 105, 109, 108, 0, 107, 0, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 77, 76, 0, 75, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 0, 0, 101, 121, 104, 0, 103, 0, 0, 0, 102, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 89, 72, 0, 71, 0, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 122, 100, 0, 99, 0, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 90, 68, 0, 67, 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 126, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500};
int chords[1024] = {0, 116, 115, 0, 114, 0, 0, 0, 111, 39, 62, 0, 112, 0, 113, 0, 32, 46, 58, 93, 59, 0, 0, 125, 44, 0, 9, 0, 91, 0, 123, 45, 0, 84, 83, 0, 82, 0, 0, 0, 79, 0, 0, 0, 80, 0, 81, 0, 10, 0, 0, 0, 0, 0, 0, 0, 63, 47, 0, 0, 0, 0, 0, 0, 110, 117, 118, 0, 119, 0, 0, 0, 120, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 78, 85, 86, 0, 87, 0, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 64, 92, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 105, 109, 108, 0, 107, 0, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 77, 76, 0, 75, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 0, 0, 101, 121, 104, 0, 103, 0, 0, 0, 102, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 89, 72, 0, 71, 0, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 122, 100, 0, 99, 0, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 90, 68, 0, 67, 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 126, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500};

void setup() {
  for (int i=0; i<10; i++) {
    pinMode(pins[i], INPUT_PULLUP);
  }
  Keyboard.begin();
}

void loop() {
  checkKeys();
  if (keyPressed()) {
    waitForRelease();
  } else {
    waiting = true;
  }
  
}

void checkKeys() {
  for (int i=0; i<10; i++) {
    int keyState = digitalRead(pins[i]);
    if (keyState == HIGH) {
      keyStatus[i] = 0;
    } else {
      keyStatus[i] = 1;
    }
  }
}

void checkKeys2() {
  for (int i=0; i<10; i++) {
    int keyState = digitalRead(pins[i]);
    if (keyState == HIGH) {
      keyStatus2[i] = 0;
    } else {
      keyStatus2[i] = 1;
    }
  }
}

void waitForRelease() {
  checkKeys();
  delay(10);
  checkKeys2();
  boolean released = oneToZero();
  while (!released) {
    checkKeys();
    delay(10);
    checkKeys2();
    released = oneToZero();
  }
  if (waiting) recordChord();
  waiting = false;
  delay(10);
}

void recordChord() {
  int ch = convert_bin2dec();
  int toType = chords[ch];
  if (toType < 256) {
    Keyboard.write(toType);
  } else {
    
    if (toType == 500) {
      alt = !alt;
      Keyboard.press(ctrlKey);
      delay(100);
      Keyboard.releaseAll();
    }    
    
  }
}

boolean keyPressed() {
  boolean kp = false;
  for (int i=0; i<10; i++) {
    if (keyStatus[i] == 1) kp = true;
  }
  return kp;
}

boolean oneToZero() {
  boolean released = false;
  for (int i=0; i<10; i++) {
    if (keyStatus[i] == 1 && keyStatus2[i] == 0) {
      released = true;
    }
  }
  return released;
}

int convert_bin2dec() {
    int val = 0;
    for ( int i = 0; i<=9 ; ++i ) {
        val = (val << 1) | keyStatus[i];
    }
    return val;
}

I experimented with a number of possible solutions involving timing windows in which a chord would be detected. However, I eventually determined that the best solution would involve detecting the chord upon key release rather than key press. The sketch above waits for any key to be released, then records the chord detected immediately prior to release.

Note that there are three arrays named “chords”—the first two are commented out. Unfortunately, the Arduino Micro’s limited storage capacity could not accommodate more than one 1024-unit integer array of chords at a time. Thus, switching between potential chord layouts required uploading a new sketch to the Arduino each time.

After developing the Arduino software, I updated the typing tutor game for use with the keyboard. Rather than a timed animation, I changed the code so that the text cursor doesn’t advance to the next letter until the prior letter has been typed. Additionally, I implemented a score system based on chord accuracy, a hint screen that pops up for 3 seconds after the grave accent (a.k.a. backtick: ‘`’, which is currently left and right ring and middle fingers together) is typed, and an “easy mode” in which the hint screen is displayed constantly and score is not kept.

After receiving feedback on my initial chord layout in class, I decided to try a new layout that included more two-finger chords (rather than three- and four-finger chords), with more coordination between right and left hands. Here is the raw JSON file for this layout:

{"\b": 640, " ": 16, "$": 240, "(": 560, ",": 24, "0": 656, "4": 272, "8": 528, "<": 320, "@": 112, "D": 546, "H": 290, "L": 162, "P": 44, "T": 33, "X": 97, "\\": 113, "`": 390, "d": 514, "h": 258, "l": 130, "p": 12, "t": 1, "x": 65, "|": 252, "#": 176, "'": 9, "+": 88, "/": 57, "3": 208, "7": 464, ";": 20, "?": 56, "C": 548, "G": 292, "K": 164, "O": 40, "S": 34, "W": 47, "[": 28, "_": 976, "c": 516, "g": 260, "k": 132, "o": 8, "s": 2, "w": 15, "{": 30, "\n": 48, "\"": 576, "&": 432, "*": 496, ".": 17, "2": 144, "6": 400, ":": 18, ">": 10, "B": 552, "F": 296, "J": 168, "N": 96, "R": 36, "V": 39, "Z": 545, "^": 368, "b": 520, "f": 264, "j": 136, "n": 64, "r": 4, "v": 7, "z": 513, "~": 585, "\t": 26, "!": 688, "%": 304, ")": 624, "-": 31, "1": 80, "5": 336, "9": 592, "=": 120, "A": 544, "E": 288, "I": 160, "M": 161, "Q": 46, "U": 35, "Y": 289, "]": 19, "a": 512, "e": 256, "i": 128, "m": 129, "q": 14, "u": 3, "y": 257, "}": 23}

As with the initial version, each typed character has a corresponding integer, which is translated into a 10-digit binary number corresponding to the 10-finger chord that must be typed.

I tested this layout extensively, and found that test subjects preferred it (almost) unanimously to the initial layout. The exact reasons varied, but I observed that individuals had an easier time typing two-finger chords than chords that involved three or more fingers. Typing speed was also 50% faster on average compared to the initial layout.

Accounting for these observations, I set out to devise another improved layout that would incorporate even more two-finger chords. In the prior layout, the letters  ‘V’ and ‘W’ still involved three- and four-finger combinations. In this layout, all letters except for ‘P’ and ‘Q’ involve two-finger combinations with the left and right hands together.

{"\b": 640, " ": 16, "$": 240, "(": 560, ",": 24, "0": 656, "4": 272, "8": 528, "<": 320, "@": 112, "D": 546, "H": 290, "L": 162, "P": 44, "T": 33, "X": 104, "\\": 113, "`": 390, "d": 514, "h": 258, "l": 130, "p": 12, "t": 1, "x": 72, "|": 252, "#": 176, "'": 9, "+": 88, "/": 57, "3": 208, "7": 464, ";": 20, "?": 56, "C": 548, "G": 292, "K": 164, "O": 40, "S": 34, "W": 100, "[": 28, "_": 976, "c": 516, "g": 260, "k": 132, "o": 8, "s": 2, "w": 68, "{": 30, "\n": 48, "\"": 576, "&": 432, "*": 496, ".": 17, "2": 144, "6": 400, ":": 18, ">": 10, "B": 552, "F": 296, "J": 168, "N": 96, "R": 36, "V": 98, "Z": 545, "^": 368, "b": 520, "f": 264, "j": 136, "n": 64, "r": 4, "v": 66, "z": 513, "~": 585, "\t": 26, "!": 688, "%": 304, ")": 624, "-": 31, "1": 80, "5": 336, "9": 592, "=": 120, "A": 544, "E": 288, "I": 160, "M": 161, "Q": 46, "U": 97, "Y": 289, "]": 19, "a": 512, "e": 256, "i": 128, "m": 129, "q": 14, "u": 65, "y": 257, "}": 23}

If I move forward with the Stenogloves, the layout above is most likely what I will integrate. Much work remains on the punctuation marks, which I am not yet satisfied with. Backspace, in particular, involves an awkward two-finger combination with the left hand, and that can be improved with the real estate gained from the new keyboard layout. In a new version of the keyboard layout, most of the punctuation marks would, in fact, resemble the alphabetical characters from the initial version of the layout (simple two- and three-finger chords).

For me, the most interesting part was going to be the proposed “typing mode” for the gloves. So, I’m going to focus on that part alone—making a pair of general-purpose typing gloves, or “stenogloves” as I’ve begun calling them.

My first step was to develop a chorded, 10-key typing system and a simple typing tutor game to learn the system. To accomplish this, I examined the Google Ngram data on English letter frequency. With over 3.5 trillion letters in the data set, here are the frequency counts for each letter:

[Chart via Peter Norvig]

At first, I attempted to create the typing system using the simplest one and two-finger chords, and mapping the letters to chords in descending order of chord difficulty. (Single-finger chords for the most common letters, simple two-finger chords for less common letters, more complex two-finger and three-finger chords for even less common letters, etc.) After creating this initial draft of the typing system, I attempted to mime my way through the alphabet, only to discover that such a system would be incredibly difficult to learn.

The system needed a common reference point—ideally, one that would allow for a mnemonic that could make learning the system easy—so I decided to try an alphabetical orientation. In this scheme, the eight most common letters in alphabetical order—’A’, ‘E’, ‘I’, ‘N’, ‘O’, ‘R’, ‘S’, ‘T’, which together account for 65% of all keystrokes—would be mapped to single-finger chords with each of the eight fingers on both hands (excluding thumbs), in sequential order from left pinky to right pinky with palms facing down. Until the letter ‘T’, letters in between these eight key letters would be typed by adding an appropriate number of fingers after the single-finger chord. (For example, ‘A’ would be left pinky, ‘B’ would be left pinky + left ring fingers, ‘C’ would be left pinky + left ring + left middle, and so on.) After the ‘T’ chord, the system continues in the opposite direction, with right pinky + right ring = ‘U’, and so on. ‘X’, ‘Y’, and ‘Z’ have special chords (left index + right index, left middle + right index, left ring + right index) because of where they fall in the alphabet. Left thumb is reserved for shift, and right thumb is reserved for space / number / punctuation.

After creating this system, I found I could mime my way through the alphabet very quickly and easily, which should provide some indicator of the difficulty with which such a system could be learned. I also added chords for numbers 0-9 and every punctuation mark, turned all the chords into 10-digit binary numbers, and converted these numbers to integers to allow them to be read easily into any computer program as a JSON file.

Here is the Python script I used to generate that JSON file:

import json

aToZ = {'A': ['LP'],
		'B': ['LP','LR'],
		'C': ['LP','LR','LM'],
		'D': ['LP','LR','LM','LI'],
		'E': ['LR'],
		'F': ['LR','LM'],
		'G': ['LR','LM','LI'],
		'H': ['LR','LM','LI','RI'],
		'I': ['LM'],
		'J': ['LM','LI'],
		'K': ['LM','LI','RI'],
		'L': ['LM','LI','RI','RM'],
		'M': ['LM','LI','RI','RM','RR'],
		'N': ['LI'],
		'O': ['RI'],
		'P': ['RI','RM'],
		'Q': ['RI','RM','RR'],
		'R': ['RM'],
		'S': ['RR'],
		'T': ['RP'],
		'U': ['RR','RP'],
		'V': ['RM','RR','RP'],
		'W': ['RI','RM','RR','RP'],
		'X': ['LI','RI'],
		'Y': ['LM','RI'],
		'Z': ['LR','RI']}

aToZbin = {'a':[],'b':[],'c':[],'d':[],'e':[],'f':[],'g':[],'h':[],'i':[],'j':[],'k':[],'l':[],'m':[],'n':[],'o':[],'p':[],'q':[],'r':[],'s':[],'t':[],'u':[],'v':[],'w':[],'x':[],'y':[],'z':[]}

fingers = ['LP', 'LR', 'LM', 'LI', 'LT', 'RT', 'RI', 'RM', 'RR', 'RP']

# lower case letters
for key, value in aToZ.iteritems():
	for finger in fingers:
		if finger in value:
			aToZbin[key.lower()].append(1)
		else:
			aToZbin[key.lower()].append(0)

# capital letters
for key in aToZbin.keys():
	l = aToZbin[key]
	m = l[:]
	m[4] = 1
	aToZbin[key.upper()] = m

# numbers 0-9
aToZbin[0] = [1,0,1,0,0,1,0,0,0,0]
aToZbin[1] = [0,0,0,1,0,1,0,0,0,0]
aToZbin[2] = [0,0,1,0,0,1,0,0,0,0]
aToZbin[3] = [0,0,1,1,0,1,0,0,0,0]
aToZbin[4] = [0,1,0,0,0,1,0,0,0,0]
aToZbin[5] = [0,1,0,1,0,1,0,0,0,0]
aToZbin[6] = [0,1,1,0,0,1,0,0,0,0]
aToZbin[7] = [0,1,1,1,0,1,0,0,0,0]
aToZbin[8] = [1,0,0,0,0,1,0,0,0,0]
aToZbin[9] = [1,0,0,1,0,1,0,0,0,0]



# symbols !-)
num_symbols = ['!', '@', '#', '$', '%', '^', '&', '*', '(', ')']
for key in aToZbin.keys():
	if key in range(10):
		l = aToZbin[key]
		m = l[:]
		m[4] = 1
		aToZbin[num_symbols[key]] = m

# space and return
aToZbin[' '] = [0,0,0,0,0,1,0,0,0,0]
aToZbin['\n'] = [0,0,0,0,1,1,0,0,0,0]

# / ?
aToZbin['/'] = [0,0,0,0,1,1,1,0,0,1]
aToZbin['?'] = [0,0,0,0,1,1,1,0,0,0]

# = +
aToZbin['='] = [0,0,0,1,1,1,1,0,0,0]
aToZbin['+'] = [0,0,0,1,0,1,1,0,0,0]

# < >
aToZbin['<'] = [0,1,0,1,0,0,0,0,0,0]
aToZbin['>'] = [0,0,0,0,0,0,1,0,1,0]

# [ ]
aToZbin['['] = [0,0,0,0,0,1,1,1,0,0]
aToZbin[']'] = [0,0,0,0,0,1,0,0,1,1]

# { }
aToZbin['{'] = [0,0,0,0,0,1,1,1,1,0]
aToZbin['}'] = [0,0,0,0,0,1,0,1,1,1]

# " '
aToZbin['\"'] = [1,0,0,1,0,0,0,0,0,0]
aToZbin['\''] = [0,0,0,0,0,0,1,0,0,1]

# , ; : . (and +space)
aToZbin[','] = [0,0,0,0,0,1,1,0,0,0]
aToZbin[';'] = [0,0,0,0,0,1,0,1,0,0]
aToZbin[':'] = [0,0,0,0,0,1,0,0,1,0]
aToZbin['.'] = [0,0,0,0,0,1,0,0,0,1]
aToZbin[', '] = [1,0,0,0,0,1,1,0,0,0]
aToZbin['; '] = [1,0,0,0,0,1,0,1,0,0]
aToZbin[': '] = [1,0,0,0,0,1,0,0,1,0]
aToZbin['. '] = [1,0,0,0,0,1,0,0,0,1]

# underscore, dash, ndash, mdash
aToZbin['_'] = [1,1,1,1,0,1,0,0,0,0]
aToZbin['-'] = [0,0,0,0,0,1,1,1,1,1]
aToZbin[u"\u2013"] = [0,0,1,1,0,1,1,1,0,0]
aToZbin[u"\u2014"] = [0,1,1,1,0,1,1,1,1,0]

# \ |
aToZbin['\\'] = [0,0,0,1,1,1,0,0,0,1]
aToZbin['|'] = [0,0,1,1,1,1,1,1,0,0]

# ~ `
aToZbin['~'] = [1,0,0,1,0,0,1,0,0,1]
aToZbin['`'] = [0,1,1,0,0,0,0,1,1,0]


# print and test uniqueness

print aToZbin

jb = aToZbin.values()

print len(jb)

unique_jb = []
duplicate = []

for i in jb:
	if i not in unique_jb:
		unique_jb.append(i)
	else:
		duplicate.append(i)

print len(unique_jb)

print duplicate


# Turn into binary

chords = {}
lookup = [[] for _ in range(1024)]

for key, value in aToZbin.iteritems():
	number = int(''.join([str(i) for i in value]), 2)
	chords[unicode(key)] = number
	lookup[number].append(key)

print chords
print lookup

with open('data.txt', 'w') as outfile:
	json.dump(chords, outfile)

And here’s the JSON file:

{": ": 530, "\u2014": 478, " ": 16, "$": 240, "(": 560, ",": 24, "0": 656, "4": 272, "8": 528, "<": 320, "@": 112, "D": 992, "H": 488, "L": 236, "P": 44, "T": 33, "X": 104, "\\": 113, "`": 390, "d": 960, "h": 456, "l": 204, "p": 12, "t": 1, "x": 72, "|": 252, "\u2013": 220, "#": 176, "'": 9, "+": 88, "/": 57, "3": 208, "7": 464, ";": 20, "?": 56, "C": 928, "G": 480, "K": 232, "O": 40, "S": 34, "; ": 532, "W": 47, "[": 28, "_": 976, "c": 896, "g": 448, "k": 200, "o": 8, "s": 2, "w": 15, "{": 30, "\n": 48, "\"": 576, "&": 432, ". ": 529, "*": 496, ".": 17, "2": 144, "6": 400, ":": 18, ">": 10, "B": 800, "F": 416, "J": 224, "N": 96, "R": 36, "V": 39, "Z": 296, "^": 368, "b": 768, "f": 384, "j": 192, "n": 64, "r": 4, "v": 7, "z": 264, "~": 585, "!": 688, "%": 304, ", ": 536, ")": 624, "-": 31, "1": 80, "5": 336, "9": 592, "=": 120, "A": 544, "E": 288, "I": 160, "M": 238, "Q": 46, "U": 35, "Y": 168, "]": 19, "a": 512, "e": 256, "i": 128, "m": 206, "q": 14, "u": 3, "y": 136, "}": 23}

Using this data, I made a simple typing tutor game in Processing that pulls the text from any news article on the web for users to type out. The code is available on Github.

The next step was to begin tinkering with actual gloves, and so I purchased a pair of inexpensive motorcycle gloves to experiment with.

I also needed to make a decision about the actuation method for each fingertip. I settled on using mechanical keyboard switches instead of force-sensitive resistors because I knew the switches would be easier to work with and would provide better tactile feedback for users. After doing a significant amount of research on mechanical keyboard components, I settled on Cherry MX Blue switches, due to their tactile feel and clicky responsiveness.

Here is a cross sectional gif of a Cherry MX Blue switch:

Tom Igoe suggested I build a simple keyboard before attaching keys to the gloves. However, I was eager to begin working with the gloves, so I turned the right one into a mouse glove using parts from a wireless mouse I purchased. Next, I plan to mount an accelerometer on the left glove, then mount keyboard switches to the fingertips on both gloves.

After playtesting the mouse glove, I built a 10-key keyboard by mounting Cherry MX Blue switches to a wooden board. I still need to connect the switches to an Arduino in order to test this keyboard, which I hope to do very soon.

It all began with an observation I made while riding the subway in the SF Bay Area. I saw a young woman using a stenographer’s keyboard on the subway, typing very quickly. I did not want to eavesdrop, so I could not determine what she was doing on the keyboard, but she was clearly producing a lot of output. Since then, I’ve been interested in the question of whether certain individuals who perform large amounts of typing on QWERTY keyboards could benefit from the use of stenographer’s (chorded) keyboards.

More recently, I had a long discussion with my classmate, Tigran Paravyan, about graphical user interfaces, and particularly the possibility of a three-dimensional interface. The interface, as we discussed it, would be for browsing the internet, but could work for browsing files on a personal computer as well. It would exist as a forest of trees, with each tree representing a browsing session that could be returned to at a later time.

Beneath each tree, on the ground, the current browsing session would be projected. The branches of the tree could contain ornaments with files or pages linked to from the current page, with progressively more remote files or pages on higher branches of the trees. The user’s browsing history would be displayed in similar structures in the roots of the trees. A user would be able to walk, climb, dig, or fly through the environment as necessary to view the desired files or pages.

The input device we discussed would be a glove with force-sensitive resistors (FSRs) in the finger tips. A user could touch her thumb to her index finger (or press her index finger on her leg, or on a table) to perform one of the four actions (walk, climb, dig, or fly), and her three other fingers (thumbs excluded) would be mapped to the three other actions. Speed of movement would be determined be the pressure a user applies to each pad. Turning could be accomplished by pressing the left hand’s pad(s) or right hand’s pad(s) independently, pressing both at the same time to move forward. Tilt sensors or accelerometers in the gloves could be mapped to other actions, such as zooming in on a particular file or page.

Alternatively, “walking” with one’s hands could translate to walking (like in the speculative video game in the movie Her—except with more tapping—see clip below), and one of the finger pads could be mapped to an auxiliary action.

NSFW LANGUAGE WARNING

Below are various notes and sketches I made in my notepad to outline this project. I plan to discuss it with Tom Igoe on Thursday and with Daniel Shiffman shortly thereafter.

Edit: Adding (low torque) servos and stiff metal plates would result in gloves that could “feel” virtual objects. Also, I purchased this pair of motorcycle gloves to begin working with.

Here are some drawings from my notebook:

The controller interfaces with a Processing sketch (the game) via an Arduino through serial communication. The player navigates a dog through a maze of cats, and when he or she touches a wall, a series of scary images and noises are displayed. If the player’s heart rate becomes elevated, the analog stick in the stuffed cat’s paw becomes more sensitive and difficult to control.

My midterm project will be a chess set that generates music and ejects pieces from the board based on Stockfish chess engine analytics. My eventual plan is to implement a physical (hardware) version of the chess set, using magnets in the pieces, Hall Effect sensors in the board, and solenoids beneath the board. However, I may rely on a software version (a chess GUI rather than a physical board) as my initial prototype. Such a version would still be connected to a physical board with solenoids beneath it to demonstrate that aspect of the project.

COMPOSITION

The chess board will be connected to the Stockfish chess engine — the world’s most powerful chess engine, which also happens to be open source. The engine will provide real-time analytics for games-in-progress, providing a score (above 0 if white is winning, below 0 if black is winning), along with the “best move” from any given board position. Mapping these variables to music will provide auditory feedback for players, turning an otherwise normal game of chess into “advanced chess” (chess where both players have access to engine analytics), but without the traditional chess engine interface. The solenoids beneath the board will provide an element of surprise and a unique way to signal that the game has ended, due to one player coming within range of a checkmate.

CONTEXT

Creating an auditory interface for the game of chess could have interesting consequences, both for chess itself and the possibility of applying such an interface to other games. I am not sure how auditory feedback will effect the game, but I hope it will make players more acutely aware of their relative strategic positions at all times. Ideally, it would provide an avenue for improvement by helping people think more like the computer chess engines.

BILL OF MATERIALS

Chess board & housings for Hall Effect sensors
64 Hall Effect sensors
32 (or more) magnets
4 solenoids
1 Arduino Mega
1 Raspberry Pi
16 multiplexor ICs
64 LEDs (if “best move” feature implemented)

TECHNICAL DRAWINGS & IMAGES

Initial Drawing (with conductive pads instead of hall effect sensors):

Rendering of Hall Effect Sensor Enclosure (for laser cutter):

Hall Effect Sensor Enclosure Prototype:

Chess GUI (software version):

SIGNAL CHAIN

Magnets >> Hall Effect Sensors >> Multiplexors >> Arduino >> Raspberry Pi (>> Music) >> Arduino >> Multiplexors >> Solenoids/LEDs