Implement <form> support, fix Youtube search

3 files changed, 180 insertions, 139 deletions

diff --git a/src/webvicli/webvicli/client.py b/src/webvicli/webvicli/client.py
index 9247fa1..9d137be 100644
--- a/src/webvicli/webvicli/client.py
+++ b/src/webvicli/webvicli/client.py
@@ -36,7 +36,7 @@ from optparse import OptionParser
 from ConfigParser import RawConfigParser
 from urlparse import urlparse
 from StringIO import StringIO
-from . import menu
+from menuparser import MenuParser
 
 VERSION = '0.5.0'
 WEBVI_STREAM_USER_AGENT = "Mozilla/5.0"
@@ -68,11 +68,6 @@ def safe_filename(name, vfat):
 
     return res
 
-def get_content_unicode(node):
-    """node.getContent() returns an UTF-8 encoded sequence of bytes (a
-    string). Convert it to a unicode object."""
-    return unicode(node.getContent(), 'UTF-8', 'replace')
-
 def guess_video_extension(mimetype, url):
     """Return extension for a video at url with a given mimetype.
 
@@ -258,115 +253,6 @@ class WVClient:
             now = datetime.datetime.now()
             self.alarm = now + datetime.timedelta(milliseconds=timeout_ms)
 
-    def parse_page(self, page):
-        if page is None:
-            return None
-        try:
-            doc = libxml2.parseDoc(page)
-        except libxml2.parserError:
-            return None
-
-        root = doc.getRootElement()
-        if root.name != 'wvmenu':
-            return None
-        queryitems = []
-        menupage = menu.Menu()
-        node = root.children
-        while node:
-            if node.name == 'title':
-                menupage.title = get_content_unicode(node)
-            elif node.name == 'ul':
-                li_node = node.children
-                while li_node:
-                    if li_node.name == 'li':
-                        menuitem = self.parse_link(li_node)
-                        menupage.add(menuitem)
-                    li_node = li_node.next
-                
-            # elif node.name == 'link':
-            #     menuitem = self.parse_link(node)
-            #     menupage.add(menuitem)
-            # elif node.name == 'textfield':
-            #     menuitem = self.parse_textfield(node)
-            #     menupage.add(menuitem)
-            #     queryitems.append(menuitem)
-            # elif node.name == 'itemlist':
-            #     menuitem = self.parse_itemlist(node)
-            #     menupage.add(menuitem)
-            #     queryitems.append(menuitem)
-            # elif node.name == 'textarea':
-            #     menuitem = self.parse_textarea(node)
-            #     menupage.add(menuitem)
-            # elif node.name == 'button':
-            #     menuitem = self.parse_button(node, queryitems)
-            #     menupage.add(menuitem)
-            node = node.next
-        doc.freeDoc()
-        return menupage
-
-    def parse_link(self, node):
-        label = ''
-        ref = None
-        is_stream = False
-        child = node.children
-        while child:
-            if child.name == 'a':
-                label = get_content_unicode(child)
-                ref = child.prop('href')
-                is_stream = child.prop('class') != 'webvi'
-            child = child.next
-        return menu.MenuItemLink(label, ref, is_stream)
-
-    def parse_textfield(self, node):
-        label = ''
-        name = node.prop('name')
-        child = node.children
-        while child:
-            if child.name == 'label':
-                label = get_content_unicode(child)
-            child = child.next
-        return menu.MenuItemTextField(label, name)
-
-    def parse_textarea(self, node):
-        label = ''
-        child = node.children
-        while child:
-            if child.name == 'label':
-                label = get_content_unicode(child)
-            child = child.next
-        return menu.MenuItemTextArea(label)
-
-    def parse_itemlist(self, node):
-        label = ''
-        name = node.prop('name')
-        items = []
-        values = []
-        child = node.children
-        while child:
-            if child.name == 'label':
-                label = get_content_unicode(child)
-            elif child.name == 'item':
-                items.append(get_content_unicode(child))
-                values.append(child.prop('value'))
-            child = child.next
-        return menu.MenuItemList(label, name, items, values, sys.stdout)
-
-    def parse_button(self, node, queryitems):
-        label = ''
-        submission = None
-        encoding = 'utf-8'
-        child = node.children
-        while child:
-            if child.name == 'label':
-                label = get_content_unicode(child)
-            elif child.name == 'submission':
-                submission = get_content_unicode(child)
-                enc = child.hasProp('encoding')
-                if enc is not None:
-                    encoding = get_content_unicode(enc)
-            child = child.next
-        return menu.MenuItemSubmitButton(label, submission, queryitems, encoding)
-
     def execute_webvi(self, request):
         """Call self.webvi.process_some until request is finished."""
         while True:
@@ -397,7 +283,7 @@ class WVClient:
             print 'Download failed:', err
             return (status, err, None)
 
-        return (status, err, self.parse_page(dlbuffer.getvalue()))
+        return (status, err, MenuParser().parse_page(dlbuffer.getvalue()))
 
     def get_quality_params(self, videosite, streamtype):
         params = []
@@ -429,7 +315,7 @@ class WVClient:
             print 'Download failed:', err
             return (None, None)
 
-        menu = self.parse_page(dlbuffer.getvalue())
+        menu = MenuParser().parse_page(dlbuffer.getvalue())
         if menu is None or len(menu) == 0:
             print 'Failed to parse menu'
             return (None, None)
diff --git a/src/webvicli/webvicli/menu.py b/src/webvicli/webvicli/menu.py
index 509ba2c..886a516 100644
--- a/src/webvicli/webvicli/menu.py
+++ b/src/webvicli/webvicli/menu.py
@@ -21,6 +21,13 @@ import urllib
 
 LINEWIDTH = 72
 
+def URI_template_substitute(template, substitutions):
+    res = template
+    for key, value in substitutions.iteritems():
+        keytemplate = '{' + str(key) + '}'
+        res = res.replace(keytemplate, str(value).replace(' ', '+'))
+    return res
+
 class Menu:
     def __init__(self):
         self.title = None
@@ -45,9 +52,13 @@ class Menu:
     def __len__(self):
         return len(self.items)
 
-    def add(self, menuitem):
+    def append(self, menuitem):
         self.items.append(menuitem)
 
+    def extend(self, menuitems):
+        for item in menuitems:
+            self.append(item)
+
 
 class MenuItemLink:
     def __init__(self, label, ref, is_stream):
@@ -119,6 +130,10 @@ class MenuItemList:
                              initial_indent=lab,
                              subsequent_indent=' '*len(lab))
 
+    def add_value(self, value, label):
+        self.items.append(label)
+        self.values.append(value)
+
     def get_query(self):
         if (self.current >= 0) and (self.current < len(self.items)):
             return {self.name: self.values[self.current]}
@@ -145,12 +160,12 @@ class MenuItemList:
 
 
 class MenuItemSubmitButton:
-    def __init__(self, label, baseurl, subitems, encoding):
+    def __init__(self, label, uritemplate, subitems, encoding):
         self.label = label
-        if type(baseurl) == unicode:
-            self.baseurl = baseurl.encode('utf-8')
+        if type(uritemplate) == unicode:
+            self.uritemplate = uritemplate.encode('utf-8')
         else:
-            self.baseurl = baseurl
+            self.uritemplate = uritemplate
         self.subitems = subitems
         self.encoding = encoding
 
@@ -158,20 +173,11 @@ class MenuItemSubmitButton:
         return '[' + self.label + ']'
 
     def activate(self):
-        baseurl = self.baseurl
-        if baseurl.find('?') == -1:
-            baseurl += '?'
-        else:
-            baseurl += '&'
-
-        parts = []
-        for sub in self.subitems:
-            for key, val in sub.get_query().iteritems():
-                try:
-                    parts.append('subst=%s,%s' % \
-                      (urllib.quote(key.encode(self.encoding, 'ignore')),
-                       urllib.quote(val.encode(self.encoding, 'ignore'))))
-                except LookupError:
-                    pass
-
-        return baseurl + '&'.join(parts)
+        substitutions = {}
+        for item in self.subitems:
+            for key, value in item.get_query().iteritems():
+                if type(value) == unicode:
+                    value = value.encode('UTF-8')
+                substitutions[str(key)] = str(value)
+        
+        return URI_template_substitute(self.uritemplate, substitutions)
diff --git a/src/webvicli/webvicli/menuparser.py b/src/webvicli/webvicli/menuparser.py
new file mode 100644
index 0000000..04aab84
--- /dev/null
+++ b/src/webvicli/webvicli/menuparser.py
@@ -0,0 +1,149 @@
+import sys
+import libxml2
+from menu import Menu, MenuItemLink, MenuItemTextField, MenuItemList, MenuItemSubmitButton
+
+def get_content_unicode(node):
+    """node.getContent() returns an UTF-8 encoded sequence of bytes (a
+    string). Convert it to a unicode object."""
+    return unicode(node.getContent(), 'UTF-8', 'replace')
+
+class InputElements:
+    def __init__(self):
+        self.elements = []
+
+    def find_by_name(self, name):
+        for elem in self.elements:
+            if elem.name == name:
+                return elem
+        return None
+        
+    def append(self, inputtype, name, mainlabel, value, label, formurl):
+        item = self.find_by_name(name)
+        if item:
+            if (inputtype == 'radio') and hasattr(item, 'add_value'):
+                item.add_value(value, label)
+            else:
+                print 'Unexpected duplicate %s element with name %s' % \
+                  (inputtype, name)
+                return
+        else:
+            # was not already in the list, create a new menu item
+            if inputtype == 'text':
+                item = MenuItemTextField(mainlabel, name)
+            elif inputtype == 'radio':
+                item = MenuItemList(mainlabel, name, [label], [value], sys.stdout)
+            elif inputtype == 'submit':
+                item = MenuItemSubmitButton(value, formurl, None, 'UTF-8')
+            
+            self.elements.append(item)
+
+    def to_menuitems(self):
+        submitbuttons = []
+        selectors = []
+        for elem in self.elements:
+            if isinstance(elem, MenuItemSubmitButton):
+                submitbuttons.append(elem)
+            else:
+                selectors.append(elem)
+
+        for button in submitbuttons:
+            button.subitems = selectors
+
+        return self.elements
+
+
+class MenuParser:
+    def parse_page(self, pagexml):
+        if pagexml is None:
+            return None
+        try:
+            doc = libxml2.parseDoc(pagexml)
+        except libxml2.parserError:
+            return None
+
+        root = doc.getRootElement()
+        if root.name != 'wvmenu':
+            return None
+        
+        queryitems = []
+        menupage = Menu()
+        node = root.children
+        while node:
+            if node.name == 'title':
+                menupage.title = get_content_unicode(node)
+            elif node.name == 'ul':
+                li_node = node.children
+                while li_node:
+                    if li_node.name == 'li':
+                        menuitem = self.parse_link(li_node)
+                        menupage.append(menuitem)
+                    li_node = li_node.next
+            elif node.name == 'form':
+                menupage.extend(self.parse_form(node))
+            node = node.next
+        doc.freeDoc()
+        return menupage
+
+    def parse_link(self, node):
+        label = ''
+        ref = None
+        is_stream = False
+        child = node.children
+        while child:
+            if child.name == 'a':
+                label = get_content_unicode(child)
+                ref = child.prop('href')
+                is_stream = child.prop('class') != 'webvi'
+            child = child.next
+        return MenuItemLink(label, ref, is_stream)
+
+    def parse_form(self, node):
+        formurl = node.prop('action')
+        inputs = InputElements()
+        child = node.children
+        while child:
+            if child.name == 'ul':
+                li_node = child.children
+                while li_node:
+                    if li_node.name == 'li':
+                        self.parse_form_item(li_node, inputs, formurl)
+                    li_node = li_node.next
+            child = child.next
+        return inputs.to_menuitems()
+
+    def parse_form_item(self, node, inputs, formurl):
+        inputnode = None
+        mainlabel = self.get_item_label(node)
+        child = node.children
+        while child:
+            if child.name == 'input':
+                self.parse_input(inputs, child, mainlabel, formurl=formurl)
+            elif child.name == 'label':
+                inputnode = self.get_input_node(child)
+                if inputnode:
+                    input_label = self.get_item_label(child)
+                    self.parse_input(inputs, inputnode, mainlabel, input_label)
+            child = child.next
+
+    def parse_input(self, inputs, node, mainlabel, inputlabel=None, formurl=None):
+        inputtype = node.prop('type')
+        name = node.prop('name')
+        value = node.prop('value')
+        inputs.append(inputtype, name, mainlabel, value, inputlabel, formurl)
+
+    def get_item_label(self, node):
+        child = node.children
+        label = ''
+        while child:
+            if child.type == 'text':
+                label += get_content_unicode(child)
+            child = child.next
+        return label.strip()
+        
+    def get_input_node(self, node):
+        child = node.children
+        while child:
+            if child.name == 'input':
+                return child
+            child = child.next
+        return None