I have spent the last few months completing a novel I started a long time ago and turning it into a non-linear interactive experience. For my final project in several classes, I have transferred this novel into a printer-equipped camera to make a new and different type of photographic experience.
Inside the antique camera is a Raspberry Pi with a camera module behind the lens. The flow of passages is controlled by a single, handwritten JSON file. When there is overlap between the tags detected in an image by Clarifai and the tags assigned to a passage, and the candidate passage occurs next in a storyline that has already begun, that passage is printed out. If no passage can be found, the camera prints poetry enabled by a recursive context-free grammar and constructed from words detected in the image.
This week, I am planning to add a back end component that will allow photos taken to be preserved as albums, and passages printed to be read later online. For now, here is the JSON file that controls the order of output:
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 |
{ "zero": { "tags": ["moon", "swamp", "marble", "north america", "insect", "street"], "order": 0, "next": ["story"] }, "guam_zero": { "tags": ["computer", "technology", "future", "keyboard", "politics"], "order": 0, "next": ["guam_one"] }, "guam_one": { "tags": ["computer", "technology", "future", "keyboard", "politics"], "order": 1, "next": [] }, "dream_zero": { "tags": ["dream", "dark", "night", "sleep", "bed", "bedroom", "indoors"], "order": 0, "next": ["chess_board"] }, "chess_board": { "tags": ["dream", "dark", "night", "sleep", "bed", "bedroom", "indoors"], "order": 2, "next": ["black_queen", "black_pawn", "black_king", "black_rook", "white_king", "white_knight"] }, "black_queen": { "tags": ["dream", "dark", "black", "night", "sleep", "bed", "bedroom", "indoors", "chess", "game", "queen"], "order": 3, "next": ["wake_up"] }, "black_pawn": { "tags": ["dream", "dark", "black", "night", "sleep", "bed", "bedroom", "indoors", "chess", "game", "pawn"], "order": 3, "next": ["wake_up"] }, "black_king": { "tags": ["dream", "dark", "black", "night", "sleep", "bed", "bedroom", "indoors", "chess", "game", "king"], "order": 3, "next": ["wake_up"] }, "black_rook": { "tags": ["dream", "dark", "black", "night", "sleep", "bed", "bedroom", "indoors", "chess", "game", "rook", "castle"], "order": 3, "next": ["wake_up"] }, "white_king": { "tags": ["dream", "dark", "white", "night", "sleep", "bed", "bedroom", "indoors", "chess", "game", "king"], "order": 3, "next": ["wake_up"] }, "white_knight": { "tags": ["dream", "dark", "white", "night", "sleep", "bed", "bedroom", "indoors", "chess", "game", "knight"], "order": 3, "next": ["wake_up"] }, "wake_up": { "tags": ["dream", "dark", "night", "sleep", "bed", "bedroom", "indoors"], "order": 4, "next": [] }, "forget": { "tags": ["man", "men", "boy"], "order": 0, "next": [] }, "story": { "tags": ["moon", "swamp", "marble", "north america", "insect", "night", "street", "woman", "women", "girl"], "order": 1, "next": ["miss_vest", "forget"] }, "miss_vest": { "tags": ["moon", "swamp", "marble", "north america", "insect", "night", "street", "woman", "women", "girl"], "order": 2, "next": ["envelope", "forget"] }, "envelope": { "tags": ["moon", "swamp", "marble", "north america", "insect", "night", "street", "woman", "women", "girl", "paper", "envelope", "mail"], "order": 3, "next": ["apartment", "forget"] }, "apartment": { "tags": ["moon", "swamp", "marble", "north america", "insect", "night", "street", "woman", "women", "girl", "paper", "envelope", "mail"], "order": 4, "next": ["email"] }, "email": { "tags": ["moon", "swamp", "marble", "north america", "insect", "night", "woman", "women", "girl", "paper", "envelope", "mail", "computer", "technology"], "order": 5, "next": ["match"] }, "match": { "tags": ["moon", "swamp", "marble", "north america", "insect", "night", "man", "men", "boy", "paper", "envelope", "mail", "computer", "technology"], "order": 5, "next": ["smithpoint", "morning"] }, "morning": { "tags": ["day", "sun", "bedroom", "bed", "breakfast", "morning", "dream", "dark", "night"], "order": 6, "next": ["call"] }, "call": { "tags": ["phone", "telephone", "technology", "computer"], "order": 7, "next": ["smithpoint"] }, "smithpoint": { "tags": ["moon", "swamp", "marble", "north america", "insect", "night", "man", "men", "boy", "bar", "drink", "alcohol", "wine", "beer"], "order": 8, "next": ["drive", "forget"] }, "drive": { "tags": ["moon", "swamp", "marble", "north america", "insect", "night", "man", "men", "boy", "bar", "drink", "alcohol", "wine", "beer"], "order": 9, "next": ["take_pill", "toss_pill"] }, "take_pill": { "tags": ["drug", "pill", "man", "men", "boy", "bar", "night", "drink", "alcohol", "wine", "beer"], "order": 10, "next": ["meet_stranger_drugs", "john_home"] }, "toss_pill": { "tags": ["moon", "swamp", "marble", "north america", "insect", "girl", "street", "woman", "women"], "order": 10, "next": ["meet_stranger_no_drugs"] }, "meet_stranger_drugs": { "tags": ["moon", "swamp", "marble", "north america", "insect", "night", "man", "men", "boy", "bar", "drink", "alcohol", "wine", "beer"], "order": 11, "next": ["john_home"] }, "meet_stranger_no_drugs": { "tags": ["moon", "swamp", "marble", "north america", "insect", "night", "man", "men", "boy", "bar", "drink", "alcohol", "wine", "beer"], "order": 11, "next": ["painting"] }, "painting": { "tags": ["painting", "art", "moon", "swamp", "marble", "north america", "insect", "night", "man", "men", "boy", "bar", "drink", "alcohol", "wine", "beer"], "order": 12, "next": [] }, "john_home": { "tags": ["drug", "pill", "man", "men", "boy", "bar", "night", "drink", "alcohol", "wine", "beer"], "order": 13, "next": [] } } |
And here is the code that’s currently running on the Raspberry Pi:
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 |
import RPi.GPIO as GPIO from Adafruit_Thermal import * import time import os import sys import json import picamera from clarifai.client import ClarifaiApi from pattern.en import referenced import gen # Init Clarifai os.environ["CLARIFAI_APP_ID"] = "nAT8dW6B0Oc5qA6JQfFcdIEr-CajukVSOZ6u_IsN" os.environ["CLARIFAI_APP_SECRET"] = "BnETdY6wtp8DmXIWCBZf8nE4XNPtlHMdtK0ISNJQ" clarifai_api = ClarifaiApi() # Assumes Env Vars Set # Init System Paths APP_PATH = os.path.dirname(os.path.realpath(__file__)) IMG_PATH = os.path.join(APP_PATH, 'img') TALE_PATH = os.path.join(APP_PATH, 'tales') # Init tale_dict with open(os.path.join(APP_PATH, 'tales_dict.json'), 'r') as infile: tale_dict = json.load(infile) # Seen tales seen_tales = list() # Init Camera camera = picamera.PiCamera() # Init Printer printer = Adafruit_Thermal("/dev/ttyAMA0", 9600, timeout=5) printer.boldOn() # Init GPIO # With camera pointed forward... # LEFT: 11 (button), 15 (led) # RIGHT: 13 (button), 16 (led) GPIO.setmode(GPIO.BOARD) ledPins = (15,16) butPins = (11,13) for pinNo in ledPins: GPIO.setup(pinNo, GPIO.OUT) for pinNo in butPins: GPIO.setup(pinNo, GPIO.IN, pull_up_down=GPIO.PUD_UP) # Open Grammar Dict with open(os.path.join(APP_PATH, 'weird_grammar.json'), 'r') as infile: grammar_dict = json.load(infile) def blink_left_right(count): ledLeft, ledRight = ledPins for _ in range(count): GPIO.output(ledRight, False) GPIO.output(ledLeft, True) time.sleep(0.2) GPIO.output(ledRight, True) GPIO.output(ledLeft, False) time.sleep(0.2) GPIO.output(ledRight, False) def to_lines(sentences): def sentence_to_lines(text): LL = 32 tokens = text.split(' ') lines = list() curLine = list() charCount = 0 for t in tokens: charCount += (len(t)+1) if charCount > LL: lines.append(' '.join(curLine)) curLine = [t] charCount = len(t)+1 else: curLine.append(t) lines.append(' '.join(curLine)) return '\n'.join(lines) sentence_lines = map(sentence_to_lines, sentences) return '\n\n'.join(sentence_lines) def open_tale(tale_name): with open(os.path.join(TALE_PATH, tale_name), 'r') as infile: tale_text = to_lines( filter(lambda x: x.strip(), infile.read().strip().split('\n')) ) return tale_text def pick_tale(tags, next_tales): choice = str() record = 0 for tale in tale_dict: if tale in next_tales or tale_dict[tale]['order'] == 0: score = len(set(tale_dict[tale]['tags']) & set(tags)) if tale in next_tales and score > 0 and not tale in seen_tales: score += 100 if score > record: choice = tale record = score return choice blink_left_right(5) imgCount = 1 cur_tale = str() while True: inputLeft, inputRight = map(GPIO.input, butPins) if inputLeft != inputRight: try: img_fn = str(int(time.time()*100))+'.jpg' img_fp = os.path.join(IMG_PATH, img_fn) camera.capture(img_fp) blink_left_right(3) result = clarifai_api.tag_images(open(img_fp)) tags = result['results'][0]['result']['tag']['classes'] if cur_tale: next_tales = tale_dict[cur_tale]['next'] else: next_tales = list() tale_name = pick_tale(tags, next_tales) cur_tale = tale_name if tale_name: lines_to_print = open_tale(tale_name) seen_tales.append(tale_name) else: grammar_dict["N"].extend(tags) if not inputLeft: sentences = [gen.make_polar(grammar_dict, 10, sent=0) for _ in range(10)] elif not inputRight: sentences = [gen.make_polar(grammar_dict, 10) for _ in range(10)] else: sentences = gen.main(grammar_dict, 10) lines_to_print = to_lines(sentences) prefix = '\n\n\nNo. %i\n\n'%imgCount printer.println(prefix+lines_to_print+'\n\n\n') grammar_dict["N"] = list() imgCount += 1 except: blink_left_right(15) print sys.exc_info() elif (not inputLeft) and (not inputRight): offCounter = 0 for _ in range(100): inputLeft, inputRight = map(GPIO.input, butPins) if (not inputLeft) and (not inputRight): time.sleep(0.1) offCounter += 1 if offCounter > 50: os.system('sudo shutdown -h now') else: break |
Click here for a Google Drive folder with all the passages from the novel.
