update base docker image

issue #66

.dockerignore

@@ -0,0 +1,15 @@
+__pycache__/
+*.py[cod]
+*.yaml
+*.db
+*.sublime-project
+*.sublime-workspace
+.vscode/
+.git/
+.gitignore
+Dockerfile
+.dockerignore
+snap/
+Jenkinsfile
+.gitlab-ci.yml
+contrib/

.gitlab-ci.yml

@@ -0,0 +1,56 @@
+# This file is a template, and might need editing before it works on your project.
+# To contribute improvements to CI/CD templates, please follow the Development guide at:
+# https://docs.gitlab.com/ee/development/cicd/templates.html
+# This specific template is located at:
+# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Docker.gitlab-ci.yml
+
+# Build a Docker image with CI/CD and push to the GitLab registry.
+# Docker-in-Docker documentation: https://docs.gitlab.com/ee/ci/docker/using_docker_build.html
+#
+# This template uses one generic job with conditional builds
+# for the default branch and all other (MR) branches.
+stages:
+  - build
+
+docker-build:
+  # Use the official docker image.
+  image: docker:latest
+  stage: build
+  services:
+    - docker:dind
+  before_script:
+    - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
+  # Default branch leaves tag empty (= latest tag)
+  # All other branches are tagged with the escaped branch name (commit ref slug)
+  script:
+    - |
+      if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]]; then
+        tag=""
+        echo "Running on default branch '$CI_DEFAULT_BRANCH': tag = 'latest'"
+      else
+        tag=":$CI_COMMIT_REF_SLUG"
+        echo "Running on branch '$CI_COMMIT_BRANCH': tag = $tag"
+      fi
+    - docker build --pull -t "$CI_REGISTRY_IMAGE${tag}" .
+    - docker push "$CI_REGISTRY_IMAGE${tag}"
+  # Run this job in a branch where a Dockerfile exists
+  rules:
+    - if: $CI_COMMIT_BRANCH
+      exists:
+        - Dockerfile
+
+# snap-build-arm64:
+#   stage: build
+#   tags:
+#     - snap-arm64
+#   script:
+#     - snapcraft
+#     - snapcraft upload --release=edge *.snap
+#
+# snap-build-amd64:
+#   stage: build
+#   tags:
+#     - snap-amd64
+#   script:
+#     - snapcraft
+#     - snapcraft upload --release=edge *.snap

CHANGELOG.md

@@ -4,6 +4,43 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Added
+- Docker image
+
+### Fixed
+- Self-invite when sending a PM from a pnut app
+
+## [1.3.0] - 2022-08-20
+### Added
+- Support for matrix stickers
+
+### Fixed
+- External matrix users unable to DM appservice
+- Joining a non-public channels on pnut makes matrix room public
+- Timestamps on generated events
+
+### Changed
+- Improved display name for pnut users in matrix rooms
+- Access tokens redacted from appservice logs
+
+## [1.2.0] - 2021-03-20
+### Added
+- Support of private messaging and private channels
+
+## [1.1.1] - 2021-03-05
+### Fixed
+- Pass correct image size info for image attachments
+
+## [1.1.0] - 2021-02-27
+### Added
+- Support for pnut v1 app streams
+
+## [1.0.2] - 2019-02-02
+### Fixed
+- error checking when avatar changes
+- hung thread when websocket is closed remotely
+- error checking for image size
+- error handling message redaction from moderator
 
 ## [1.0.1] - 2019-01-09
 ### Fixed
@@ -29,7 +66,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - This CHANGELOG file because I can't believe for the last year I wasn't
 keeping track of releases for this project. :p
 
-[Unreleased]: https://gitlab.dreamfall.space/thrrgilag/pnut-matrix/compare/1.0.0...HEAD
+[Unreleased]: https://gitlab.dreamfall.space/thrrgilag/pnut-matrix/compare/1.0.2...HEAD
+[1.0.2]: https://gitlab.dreamfall.space/thrrgilag/pnut-matrix/tags/1.0.2
 [1.0.1]: https://gitlab.dreamfall.space/thrrgilag/pnut-matrix/tags/1.0.1
 [1.0.0]: https://gitlab.dreamfall.space/thrrgilag/pnut-matrix/tags/1.0.0
 [0.0.1]: https://gitlab.dreamfall.space/thrrgilag/pnut-matrix/tags/v0.0.1

Dockerfile

@@ -0,0 +1,14 @@
+FROM python:3.11-slim-bookworm
+
+WORKDIR /usr/src/app
+
+RUN apt-get update && apt-get install libmagic-dev curl -y
+
+COPY . .
+RUN pip install --no-cache-dir -r requirements.txt
+
+ENV CONFIG_FILE=/data/config.yaml
+VOLUME /data
+WORKDIR /data
+EXPOSE 5000
+CMD [ "python", "/usr/src/app/pnut-matrix.py" ]

README.md

@@ -2,29 +2,65 @@
 
 This is a pnut.io channel bridge for Matrix using the Application Services (AS) API.
 
-This bridge will pass pnut.io channel messages through to Matrix, and Matrix messages through to pnut.io channels.  Currently only public chat channels on pnut.io are supported but work is in progress to support private messages.
+This bridge will pass pnut.io channel messages through to Matrix, and Matrix messages through to pnut.io channels.
 
 
 ## Usage
 
-To join a channel on pnut.io configured for public use:
+The public bridge is once again online!
 
-* `/join #pnut_ID:monkeystew.net` where ID is the id number of the channel on pnut.io
-* You can get the id number by looking at the URL when viewing the channel in patter.chat
-
-By default any messages you send will be posted as the matrixbot user on pnut.io with your display name.  You can however authorize pnut-matrix to post as your account instead.
-
-To link your Matrix ID with your pnut.io account:
-
-* Start a direct chat with `@pnut-matrix-bot:monkeystew.net`
-* Type the `!help` command to get started.
+See [Using-the-public-bridge](https://gitlab.com/thrrgilag/pnut-matrix/-/wikis/Using-the-public-bridge) for details.
 
 
-## Questions, comments, or issues
+## Installation
 
-Join the chat!
+Currently pnut-matrix has been only tested with and confirmed to work with [synapse]. Please refer to the [synapse installation instructions] for details on how to setup your homeserver.
 
-* [Matrix] [#pnut-matrix:monkeystew.net](https://matrix.to/#/#pnut-matrix:monkeystew.net)
-* [pnut.io] [patter.chat](https://patter.chat/room/999)
-* [XMPP] [pnut-matrix@chat.monkeystew.net](xmpp:pnut-matrix@chat.monkeystew.net?join)
+To install the latest version of pnut-matrix from source:
 
+```sh
+git clone https://gitlab.com/thrrgilag/pnut-matrix.git
+cd pnut-matrix
+python3 -m venv env
+source env/bin/activate
+pip3 install -r requirements.txt
+```
+
+
+## Configuration
+
+Copy `config.yaml-sample` to `config.yaml` and edit for your setup. Likewise copy `appservice.yaml-sample` to `appservice.yaml` and edit to match the tokens, prefix, and port listed in `config.yaml`. This is the configuration that synapse will need to reference so that it can connect the bridge. Make sure you modify your [syanpse configuration] accordingly.
+
+
+## Tweaks
+
+There exists a bug in synapse which prevents the bridge user from accepting DM invites from users on other homeservers. To work around the issue you can create a profile for the bot user.[^1]
+
+```sh
+curl --data '{"type": "m.login.application_service", "username": "your_sender_localpart"}' 'http://yourhomeserver/_matrix/client/r0/register?access_token=your_as_token'
+```
+
+
+## Contributing and support
+
+You can open issues for bugs or feature requests and you can submit merge requests to this project on [GitLab]. You can also submit issues and patches directly to [morgan@mcmillian.dev].
+
+Join my public chat room for development discussion.
+
+- [pnut-matrix on pnut.io]
+- [#pnut_999:pnut-matrix.dreamfall.space]
+
+
+## License
+
+GPLv3, see [LICENSE].
+
+[synapse]: https://github.com/matrix-org/synapse
+[synapse installation instructions]: https://matrix-org.github.io/synapse/latest/setup/installation.html
+[syanpse configuration]: https://matrix-org.github.io/synapse/latest/application_services.html
+[GitLab]: https://gitlab.com/thrrgilag/pnut-matrix/
+[morgan@mcmillian.dev]: mailto:morgan@mcmillian.dev
+[pnut-matrix on pnut.io]: https://patter.chat/999
+[#pnut_999:pnut-matrix.dreamfall.space]: https://matrix.to/#/#pnut_999:pnut-matrix.dreamfall.space
+[LICENSE]: LICENSE
+[^1]: https://github.com/matrix-org/matrix-appservice-irc/issues/1270#issuecomment-849765090

appservice.py

@@ -3,10 +3,12 @@ import requests
 import logging
 import re
 import pnutpy
+import textwrap
+import time
 
 from matrix_client.api import MatrixHttpApi
 from matrix_client.api import MatrixError, MatrixRequestError
-from models import Avatars, Rooms, Events, Users
+from models import Avatars, Rooms, Events, Users, DirectRooms, ControlRooms
 from database import db_session
 from sqlalchemy import and_
 from flask import Flask, jsonify, request, abort
@@ -27,6 +29,7 @@ def forbidden(error):
 def shutdown_session(exception=None):
     db_session.remove()
 
+@app.route("/_matrix/app/v1/rooms/<alias>")
 @app.route("/rooms/<alias>")
 def query_alias(alias):
     alias_localpart = alias.split(":")[0][1:]
@@ -56,13 +59,17 @@ def query_alias(alias):
             room['name'] = channel_settings['name']
         if 'description' in channel_settings:
             room['topic'] = channel_settings['description']
-        if channel.acl.read.any_user:
+        if channel.acl.read.public:
             room['preset'] = 'public_chat'
             room['visibility'] = 'public'
+        else:
+            abort(401)
 
-        url = app.config['MATRIX_HOST'] + '/_matrix/client/api/v1/createRoom?access_token='
-        url += app.config['MATRIX_AS_TOKEN']
-        headers = {"Content-Type":"application/json"}
+        url = app.config['MATRIX_HOST'] + '/_matrix/client/api/v1/createRoom'
+        headers = {
+            "Content-Type": "application/json",
+            "Authorization": "Bearer " + app.config['MATRIX_AS_TOKEN']
+        }
         r = requests.post(url, headers=headers, data=json.dumps(room))
         if r.status_code == 200:
             pnutpy.api.subscribe_channel(channel_id)
@@ -75,15 +82,25 @@ def query_alias(alias):
             db_session.add(rr)
             db_session.commit()
 
+        else:
+            logger.error("Unable to create room")
+            logger.error(r.status_code)
+            logger.error(r.text)
+            abort(400)
+
+    except pnutpy.errors.PnutPermissionDenied:
+        abort(401)
+
     except Exception:
         logger.exception("-couldn't get the pnut channel-")
         abort(404)
 
     return jsonify({})
 
+@app.route("/_matrix/app/v1/transactions/<transaction>", methods=["PUT"])
 @app.route("/transactions/<transaction>", methods=["PUT"])
 def on_receive_events(transaction):
-    
+
     access_token = request.args.get('access_token', '')
     if access_token != app.config['MATRIX_HS_TOKEN']:
         abort(403)
@@ -92,7 +109,6 @@ def on_receive_events(transaction):
     for event in events:
         logger.debug(event)
 
-        # TODO: route event if it's in the control room
         if app.config['MATRIX_ADMIN_ROOM'] and app.config['MATRIX_ADMIN_ROOM'] == event['room_id']:
             return on_admin_event(event)
 
@@ -101,16 +117,28 @@ def on_receive_events(transaction):
         if event['type'] == 'm.room.message':
             new_message(event, user)
 
+        elif event['type'] == 'm.sticker':
+            new_sticker(event, user)
+
         elif event['type'] == 'm.room.redaction':
             delete_message(event, user)
 
         elif event['type'] == 'm.room.member':
-            logger.debug("-room member event")
+            if 'is_direct' in event['content'] and 'membership' in event['content']:
+                if event['content']['membership'] == "invite" and event['content']['is_direct']:
+                    return on_direct_invite(event)
+
+            if 'membership' in event['content']:
+                if event['content']['membership'] == "leave":
+                    return on_leave_event(event)
+
+            logger.debug("----room member event----")
+            logger.debug(user)
+            logger.debug(event)
 
     return jsonify({})
 
-def new_message(event, user):
-    
+def new_sticker(event, user):
     if app.config['MATRIX_PNUT_PREFIX'] in event['user_id'] or 'pnut-bridge' in event['user_id']:
         logger.debug('-skipping dup event-')
         return
@@ -120,49 +148,92 @@ def new_message(event, user):
         logger.debug('-room not mapped-')
         return
 
-    if 'msgtype' not in event['content']:
-        logger.debug('-unknown message type-')
-        return
-    
     if user is not None:
         token = user.pnut_user_token
         prefix = ""
     else:
         token = app.config['MATRIX_PNUT_TOKEN']
-        prefix = "[" + get_displayname(event['user_id']) + "] (" + event['user_id'] + ")\n"
+        matrix_profile = get_profile(event['user_id'])
+        if "displayname" in matrix_profile:
+            prefix = "[" + matrix_profile['displayname'] + "] (" + event['user_id'] + ")\n"
+        else:
+            prefix = "(" + event['user_id'] + ")\n"
 
     pnutpy.api.add_authorization_token(token)
-    text = None
-    embed = None
-
-    if event['content']['msgtype'] == 'm.text' or event['content']['msgtype'] == 'm.notice':
-        text = event['content']['body']
-    
-    elif event['content']['msgtype'] == 'm.emote':
-        text = "* " + event['content']['body']
-
-    elif event['content']['msgtype'] == 'm.image':
-        text = event['content']['body'] + "\n"
-        text += app.config['MATRIX_HOST'] + '/_matrix/media/r0/download/' + event['content']['url'][6:]
-        embed = [raw_from_event(event)]
-
-    elif event['content']['msgtype'] == 'm.video':
-        text = event['content']['body'] + "\n"
-        text += app.config['MATRIX_HOST'] + '/_matrix/media/r0/download/' + event['content']['url'][6:]
-
-    elif event['content']['msgtype'] == 'm.audio':
-        text = event['content']['body'] + "\n"
-        text += app.config['MATRIX_HOST'] + '/_matrix/media/r0/download/' + event['content']['url'][6:]
-
-    elif event['content']['msgtype'] == 'm.file':
-        text = event['content']['body'] + "\n"
-        text += app.config['MATRIX_HOST'] + '/_matrix/media/r0/download/' + event['content']['url'][6:]
-
+    # evtext, evraw = msg_from_event(event)
+    text = "sticker::" + event['content']['body'] + "\n"
+    value = {'type': "photo", 'version': "1.0"}
+    value['url'] = app.config['MATRIX_URL'] + '/_matrix/media/r0/download/' + event['content']['url'][6:]
+    value['title'] = event['content']['body']
+    if 'h' in event['content']['info'] and 'w' in event['content']['info']:
+        value['height'] = event['content']['info']['h']
+        value['width'] = event['content']['info']['h']
     else:
-        logger.debug('-unknown msg type- ' + event['content']['msgtype'])
+        return
+    raw = {'type': "io.pnut.core.oembed", 'value': value}
+    embed = [raw]
+    text = prefix + text
+
+    try:
+        msg, meta = pnutpy.api.create_message(room.pnut_chan, data={'text': text, 'raw': embed})
+        revent = Events(
+            event_id=event['event_id'],
+            room_id=event['room_id'],
+            pnut_msg_id=msg.id,
+            pnut_user_id=msg.user.id,
+            pnut_chan_id=room.pnut_chan,
+            deleted=False
+        )
+        db_session.add(revent)
+        db_session.commit()
+
+    except pnutpy.errors.PnutAuthAPIException:
+        logger.exception('-unable to post to pnut channel-')
         return
 
-    text = prefix + text
+    except Exception:
+        logger.exception('-something bad happened here-')
+        return
+
+def new_message(event, user):
+    if app.config['MATRIX_PNUT_PREFIX'] in event['user_id'] or 'pnut-bridge' in event['user_id']:
+        logger.debug('-skipping dup event-')
+        return
+
+    if 'msgtype' not in event['content']:
+        logger.debug('-unknown message type-')
+        return
+
+    control = ControlRooms.query.filter(ControlRooms.room_id == event['room_id']).one_or_none()
+    if control is not None:
+        return on_control_message(event)
+
+    direct = DirectRooms.query.filter(DirectRooms.room_id == event['room_id']).one_or_none()
+    if direct is not None:
+        return on_direct_message(event, user, direct)
+
+    room = Rooms.query.filter(Rooms.room_id == event['room_id']).one_or_none()
+    if room is None:
+        logger.debug('-room not mapped-')
+        return
+
+    if user is not None:
+        token = user.pnut_user_token
+        prefix = ""
+    else:
+        token = app.config['MATRIX_PNUT_TOKEN']
+        matrix_profile = get_profile(event['user_id'])
+        if "displayname" in matrix_profile:
+            prefix = "[" + matrix_profile['displayname'] + "] (" + event['user_id'] + ")\n"
+        else:
+            prefix = "(" + event['user_id'] + ")\n"
+    pnutpy.api.add_authorization_token(token)
+
+    embed = [crosspost_raw(event)]
+    evtext, evraw = msg_from_event(event)
+    text = prefix + evtext
+    if len(evraw) > 0:
+        embed.append(evraw)
     try:
         msg, meta = pnutpy.api.create_message(room.pnut_chan, data={'text': text, 'raw': embed})
         revent = Events(
@@ -179,34 +250,111 @@ def new_message(event, user):
         if user is not None:
             cctag = re.search('##$', text)
             if cctag:
+                raw = []
                 cname = get_channel_settings(room.pnut_chan)['name']
                 text = text[:-2]
-                text += '\n\n[' + cname + "](https://patter.chat/room.html?channel=" + str(room.pnut_chan) + ")"
-                r, meta = pnutpy.api.create_post(data={'text': text})
+                ftext = '\n\n[' + cname + "](https://patter.chat/room.html?channel=" + str(room.pnut_chan) + ")"
+                mtext = textwrap.wrap(text + ftext, 254)
+                if len(mtext) > 1:
+                    longpost = {
+                        'title': "",
+                        'body': text,
+                        'tstamp': time.time() * 1000
+                    }
+                    pretext = textwrap.wrap(text, 100)
+                    text = pretext[0]
+                    text += "... - https://longpo.st/p/{object_id} - #longpost"
+                    raw.append({'type':"nl.chimpnut.blog.post", 'value': longpost})
+
+                text += ftext
+                r, meta = pnutpy.api.create_post(data={'text': text, 'raw': raw})
 
     except pnutpy.errors.PnutAuthAPIException:
         logger.exception('-unable to post to pnut channel-')
         return
-    
+
     except Exception:
         logger.exception('-something bad happened here-')
         return
 
-def raw_from_event(event):
+def msg_from_event(event):
+    text = None
+    raw = {}
+    if event['content']['msgtype'] == 'm.text' or event['content']['msgtype'] == 'm.notice':
+        text = event['content']['body']
 
-    url = app.config['MATRIX_HOST'] + '/_matrix/media/r0/download/' + event['content']['url'][6:]
+    elif event['content']['msgtype'] == 'm.emote':
+        text = "* " + event['content']['body']
+
+    elif event['content']['msgtype'] == 'm.image':
+        text = event['content']['body'] + "\n"
+        text += app.config['MATRIX_URL'] + '/_matrix/media/r0/download/' + event['content']['url'][6:]
+        raw = raw_from_event(event)
+
+    elif event['content']['msgtype'] == 'm.video':
+        text = event['content']['body'] + "\n"
+        text += app.config['MATRIX_URL'] + '/_matrix/media/r0/download/' + event['content']['url'][6:]
+
+    elif event['content']['msgtype'] == 'm.audio':
+        text = event['content']['body'] + "\n"
+        text += app.config['MATRIX_URL'] + '/_matrix/media/r0/download/' + event['content']['url'][6:]
+
+    elif event['content']['msgtype'] == 'm.file':
+        text = event['content']['body'] + "\n"
+        text += app.config['MATRIX_URL'] + '/_matrix/media/r0/download/' + event['content']['url'][6:]
+
+    else:
+        logger.debug('-unknown msg type- ' + event['content']['msgtype'])
+        return
+
+    return text, raw
+
+def crosspost_raw(event):
+    cross_profile = {'username': event['user_id']}
+    matrix_profile = get_profile(event['user_id'])
+    if "avatar_url" in matrix_profile:
+        cross_profile['avatar_image'] = app.config['MATRIX_URL'] + '/_matrix/media/r0/download/' + matrix_profile['avatar_url'][6:]
+
+    embed = {
+        'type': "io.pnut.core.crosspost",
+        'value': {
+            'canonical_url': "https://matrix.to/#/" + event['room_id'] + "/" + event['event_id'] + ":" + app.config['MATRIX_DOMAIN'],
+            'source': {
+                'name': "matrix",
+                'url': "https://matrix.org"
+            },
+            'user': cross_profile
+        }
+    }
+    return embed
+
+def raw_from_event(event):
+    url = app.config['MATRIX_URL'] + '/_matrix/media/r0/download/' + event['content']['url'][6:]
 
     if event['content']['msgtype'] == 'm.image':
         value = {'type': "photo", 'version': "1.0"}
         value['url'] = url
         value['title'] = event['content']['body']
         if 'info' in event['content']:
-            value['width'] = event['content']['info']['w']
-            value['height'] = event['content']['info']['h']
+            if 'w' in event['content']['info']:
+                value['width'] = event['content']['info']['w']
+            else:
+                value['width'] = 200
+            if 'h' in event['content']['info']:
+                value['height'] = event['content']['info']['h']
+            else:
+                value['height'] = 200
             if 'thumbnail_info' in event['content']['info']:
-                value['thumbnail_url'] = app.config['MATRIX_HOST'] + '/_matrix/media/r0/download/' + event['content']['info']['thumbnail_url'][6:]
-                value['thumbnail_width'] = event['content']['info']['thumbnail_info']['w']
-                value['thumbnail_height'] = event['content']['info']['thumbnail_info']['h']
+                if 'thumbnail_url' in event['content']['info']:
+                    value['thumbnail_url'] = app.config['MATRIX_URL'] + '/_matrix/media/r0/download/' + event['content']['info']['thumbnail_url'][6:]
+                    if 'w' in event['content']['info']['thumbnail_info']:
+                        value['thumbnail_width'] = event['content']['info']['thumbnail_info']['w']
+                    else:
+                        value['thumbnail_width'] = 200
+                    if 'h' in event['content']['info']['thumbnail_info']:
+                        value['thumbnail_height'] = event['content']['info']['thumbnail_info']['h']
+                    else:
+                        value['thumbnail_height'] = 200
 
     elif event['content']['msgtype'] == 'm.video':
         # TODO: Need to sort out the oembed for this media type
@@ -244,18 +392,20 @@ def delete_message(event, user):
     if e is None:
         logger.debug("- can't find the event to remove -")
         return
-    
-    r, meta = pnutpy.api.delete_message(e.pnut_chan_id, e.pnut_msg_id)
-    e.deleted = True
-    db_session.commit()
 
-def get_displayname(userid):
-    url = "http://localhost:8008/_matrix/client/r0/profile/" + userid
+    try:
+        r, meta = pnutpy.api.delete_message(e.pnut_chan_id, e.pnut_msg_id)
+        e.deleted = True
+        db_session.commit()
+
+    except pnutpy.errors.PnutPermissionDenied as e:
+        pass
+
+def get_profile(userid):
+    url = app.config['MATRIX_HOST'] + "/_matrix/client/r0/profile/" + userid
     r = requests.get(url)
     if r.status_code == 200:
-        data = json.loads(r.text)
-        if 'displayname' in data:
-            return data["displayname"]
+        return json.loads(r.text)
     return userid
 
 def get_channel_settings(channel_id):
@@ -272,36 +422,100 @@ def get_channel_settings(channel_id):
 
     return channel_settings
 
+def create_room(channel, user):
+    channel_settings = {}
+    for item in channel.raw:
+        if item.type == 'io.pnut.core.chat-settings':
+            channel_settings = item.value
+
+    # Matrix sdk doesn't include all details in a single call
+    room = {'room_alias_name': app.config['MATRIX_PNUT_PREFIX'] + channel.id}
+    logger.debug(user)
+    logger.debug(room)
+    room['invite'] = [user.matrix_id]
+    if 'name' in channel_settings:
+        room['name'] = channel_settings['name']
+    if 'description' in channel_settings:
+        room['topic'] = channel_settings['description']
+    if channel.acl.read.public:
+        room['preset'] = 'public_chat'
+        room['visibility'] = 'public'
+    elif channel.acl.read.any_user or channel.acl.read.you:
+        room['preset'] = 'private_chat'
+        room['visibility'] = 'private'
+    else:
+        abort(401)
+
+    url = app.config['MATRIX_HOST'] + '/_matrix/client/api/v1/createRoom'
+    headers = {
+        "Content-Type": "application/json",
+        "Authorization": "Bearer " + app.config['MATRIX_AS_TOKEN']
+    }
+    r = requests.post(url, headers=headers, data=json.dumps(room))
+
+    if r.status_code == 200:
+        #pnutpy.api.subscribe_channel(channel.id)
+        rdata = r.json()
+        rr = Rooms(
+            room_id=rdata['room_id'],
+            pnut_chan=channel.id,
+            portal=True
+        )
+        db_session.add(rr)
+        db_session.commit()
+
+    else:
+        logger.error("Unable to create room")
+        logger.error(r.status_code)
+        logger.error(r.text)
+        abort(400)
+
+def new_matrix_user(username):
+    matrix_api = MatrixHttpApi(app.config['MATRIX_HOST'],
+        token=app.config['MATRIX_AS_TOKEN'])
+    data = {'type': 'm.login.application_service','username': app.config['MATRIX_PNUT_PREFIX'] + username}
+    try:
+        matrix_api.register(content=data)
+
+    except Exception:
+        errmsg = "- new_matrix_user user already exists -"
+        logger.warning(errmsg)
+
 def on_admin_event(event):
-    matrix_api = MatrixHttpApi(app.config['MATRIX_HOST'], 
+    matrix_api = MatrixHttpApi(app.config['MATRIX_HOST'],
         token=app.config['MATRIX_AS_TOKEN'])
     logger.debug("- admin room event recieved -")
 
     if event['type'] != 'm.room.message':
         return jsonify({})
-    
+
     msg = event['content']['body'].split(' ')
 
-    if msg[0] == 'help':
-        if len(msg) > 1:
-            matrix_api.send_message(event['room_id'], cmd_admin_help(msg[1]))
-        else:
-            matrix_api.send_message(event['room_id'], cmd_admin_help())
+    try:
+        if msg[0] == 'help':
+            if len(msg) > 1:
+                matrix_api.send_message(event['room_id'], cmd_admin_help(msg[1]))
+            else:
+                matrix_api.send_message(event['room_id'], cmd_admin_help())
 
-    elif msg[0] == 'list':
-        matrix_api.send_message(event['room_id'], cmd_admin_list())
+        elif msg[0] == 'list':
+            matrix_api.send_message(event['room_id'], cmd_admin_list())
 
-    elif msg[0] == 'unlink':
-        if len(msg) > 1:
-            matrix_api.send_message(event['room_id'], cmd_admin_unlink(msg[1]))
-        else:
-            matrix_api.send_message(event['room_id'], cmd_admin_help('unlink'))
+        elif msg[0] == 'unlink':
+            if len(msg) > 1:
+                matrix_api.send_message(event['room_id'], cmd_admin_unlink(msg[1]))
+            else:
+                matrix_api.send_message(event['room_id'], cmd_admin_help('unlink'))
 
-    elif msg[0] == 'link':
-        if len(msg) > 2:
-            matrix_api.send_message(event['room_id'], cmd_admin_link(msg[1], msg[2]))
-        else:
-            matrix_api.send_message(event['room_id'], cmd_admin_help('link'))
+        elif msg[0] == 'link':
+            if len(msg) > 2:
+                matrix_api.send_message(event['room_id'], cmd_admin_link(msg[1], msg[2]))
+            else:
+                matrix_api.send_message(event['room_id'], cmd_admin_help('link'))
+
+    except Exception:
+        errmsg = "- on_admin_event -"
+        logger.exception(errmsg)
 
     return jsonify({})
 
@@ -357,7 +571,7 @@ def cmd_admin_list():
     return text
 
 def cmd_admin_link(room_id, pnut_chan_id):
-    matrix_api = MatrixHttpApi(app.config['MATRIX_HOST'], 
+    matrix_api = MatrixHttpApi(app.config['MATRIX_HOST'],
         token=app.config['MATRIX_AS_TOKEN'])
     pnutpy.api.add_authorization_token(app.config['MATRIX_PNUT_TOKEN'])
 
@@ -370,7 +584,7 @@ def cmd_admin_link(room_id, pnut_chan_id):
     try:
         channel, meta = pnutpy.api.subscribe_channel(pnut_chan_id)
         r = matrix_api.join_room(room_id)
-    
+
         rec = Rooms(
             room_id=room_id,
             pnut_chan=channel.id,
@@ -389,32 +603,33 @@ def cmd_admin_link(room_id, pnut_chan_id):
         logger.exception(errmsg)
         return errmsg
 
-def cmd_admin_unlink(id):
-    matrix_api = MatrixHttpApi(app.config['MATRIX_HOST'], 
+def cmd_admin_unlink(rid):
+    matrix_api = MatrixHttpApi(app.config['MATRIX_HOST'],
         token=app.config['MATRIX_AS_TOKEN'])
     pnutpy.api.add_authorization_token(app.config['MATRIX_PNUT_TOKEN'])
 
-    if id.startswith('!'):
-        room = Rooms.query.filter(Rooms.room_id == id).one_or_none()
+    if rid.startswith('!'):
+        room = Rooms.query.filter(Rooms.room_id == rid).one_or_none()
     else:
-        room = Rooms.query.filter(Rooms.pnut_chan == id).one_or_none()
+        room = Rooms.query.filter(Rooms.pnut_chan == rid).one_or_none()
 
-    if room.portal:
-        alias = "#" + app.config['MATRIX_PNUT_PREFIX']
-        alias += str(room.pnut_chan) + ":"
-        alias += app.config['MATRIX_DOMAIN']
-        matrix_api.remove_room_alias(alias)
+    if hasattr(room, 'portal'):
+        if room.portal:
+            alias = "#" + app.config['MATRIX_PNUT_PREFIX']
+            alias += str(room.pnut_chan) + ":"
+            alias += app.config['MATRIX_DOMAIN']
+            matrix_api.remove_room_alias(alias)
 
-        # Kicking users needs at least moderator privs
-        members = matrix_api.get_room_members(room.room_id)
-        reason = "Portal room has been unlinked by administrator"
-        for m in members['chunk']:
-            if m['membership'] == 'join' and m['sender'] != app.config['MATRIX_AS_ID']:
-                if room.portal:
-                    matrix_api.kick_user(room.room_id, m['sender'], reason=reason)
-                else:
-                    if m['sender'].startswith("@" + app.config['MATRIX_PNUT_PREFIX']):
+            # Kicking users needs at least moderator privs
+            members = matrix_api.get_room_members(room.room_id)
+            reason = "Portal room has been unlinked by administrator"
+            for m in members['chunk']:
+                if m['content']['membership'] == 'join' and m['sender'] != app.config['MATRIX_AS_ID']:
+                    if room.portal:
                         matrix_api.kick_user(room.room_id, m['sender'], reason=reason)
+                    else:
+                        if m['sender'].startswith("@" + app.config['MATRIX_PNUT_PREFIX']):
+                            matrix_api.kick_user(room.room_id, m['sender'], reason=reason)
 
     try:
         channel, meta = pnutpy.api.unsubscribe_channel(room.pnut_chan)
@@ -431,3 +646,272 @@ def cmd_admin_unlink(id):
         errmsg = "- error while unlinking room -"
         logger.exception(errmsg)
         return errmsg
+
+def on_direct_invite(event):
+    if event['state_key'] == app.config['MATRIX_AS_ID']:
+        matrix_api = MatrixHttpApi(app.config['MATRIX_HOST'],
+            token=app.config['MATRIX_AS_TOKEN'])
+        dm = ControlRooms(room_id=event['room_id'])
+
+    else:
+        matrix_api = MatrixHttpApi(app.config['MATRIX_HOST'],
+            identity=event['state_key'],
+            token=app.config['MATRIX_AS_TOKEN'])
+        bridge_user = event['state_key']
+        pnut_user = bridge_user.replace(app.config['MATRIX_PNUT_PREFIX'],'').split(':')[0]
+
+        user = Users.query.filter(Users.matrix_id == event['sender']).one_or_none()
+        if user is not None:
+            # TODO: need to handle if the user isn't registered
+            pnutpy.api.add_authorization_token(user.pnut_user_token)
+            channel, meta = pnutpy.api.existing_pm(ids=pnut_user)
+            new_matrix_user(pnut_user)
+
+            dm = DirectRooms(room_id=event['room_id'],
+                    bridge_user=bridge_user, pnut_chan=channel.id)
+
+    try:
+        matrix_api.join_room(event['room_id'])
+
+        db_session.add(dm)
+        db_session.commit()
+
+    except Exception:
+        errmsg = "- on_direct_invite -"
+        logger.exception(errmsg)
+
+    return jsonify({})
+
+def on_leave_event(event):
+    direct = DirectRooms.query.filter(DirectRooms.room_id == event['room_id']).one_or_none()
+    if direct is not None:
+
+        matrix_api = MatrixHttpApi(app.config['MATRIX_HOST'],
+                identity=direct.bridge_user,
+                token=app.config['MATRIX_AS_TOKEN'])
+        try:
+            matrix_api.leave_room(event['room_id'])
+            db_session.delete(direct)
+            db_session.commit()
+
+        except Exception:
+            errmsg = "- on_leave_event -"
+            logger.exception(errmsg)
+
+    control = ControlRooms.query.filter(ControlRooms.room_id == event['room_id']).one_or_none()
+    if control is not None:
+        matrix_api = MatrixHttpApi(app.config['MATRIX_HOST'],
+                token=app.config['MATRIX_AS_TOKEN'])
+        try:
+            matrix_api.leave_room(event['room_id'])
+            db_session.delete(control)
+            db_session.commit()
+
+        except Exception:
+            errmsg = "- on_leave_event -"
+            logger.exception(errmsg)
+
+    return jsonify({})
+
+def on_direct_message(event, user, room):
+    if user is not None:
+        token = user.pnut_user_token
+        prefix = ""
+    else:
+        token = app.config['MATRIX_PNUT_TOKEN']
+        matrix_profile = get_profile(event['user_id'])
+        if "displayname" in matrix_profile:
+            prefix = "[" + matrix_profile['displayname'] + "] (" + event['user_id'] + ")\n"
+        else:
+            prefix = "(" + event['user_id'] + ")\n"
+    pnutpy.api.add_authorization_token(token)
+
+    embed = [crosspost_raw(event)]
+    evtext, evraw = msg_from_event(event)
+    text = prefix + evtext
+    if len(evraw) > 0:
+        embed.append(evraw)
+    try:
+        msg, meta = pnutpy.api.create_message(room.pnut_chan, data={'text': text, 'raw': embed})
+        revent = Events(
+            event_id=event['event_id'],
+            room_id=event['room_id'],
+            pnut_msg_id=msg.id,
+            pnut_user_id=msg.user.id,
+            pnut_chan_id=room.pnut_chan,
+            deleted=False
+        )
+        db_session.add(revent)
+        db_session.commit()
+
+    except pnutpy.errors.PnutAuthAPIException:
+        logger.exception('-unable to post to pnut channel-')
+
+    except Exception:
+        logger.exception('-something bad happened here-')
+
+    return jsonify({})
+
+def on_control_message(event):
+    matrix_api = MatrixHttpApi(app.config['MATRIX_HOST'],
+        token=app.config['MATRIX_AS_TOKEN'])
+    logger.debug("- direct room event received -")
+
+    if event['type'] != 'm.room.message':
+        return jsonify({})
+
+    msg = event['content']['body'].split(' ')
+
+    try:
+        if msg[0] == '!help' or msg[0] == 'help':
+            if len(msg) > 1:
+                matrix_api.send_message(event['room_id'], cmd_user_help(msg[1]))
+            else:
+                matrix_api.send_message(event['room_id'], cmd_user_help())
+
+        elif msg[0] == '!auth':
+            matrix_api.send_message(event['room_id'], cmd_user_auth())
+
+        elif msg[0] == '!save':
+            if len(msg) > 1:
+                matrix_api.send_message(event['room_id'], cmd_user_save(event['sender'], msg[1]))
+            else:
+                matrix_api.send_message(event['room_id'], cmd_user_save())
+
+        elif msg[0] == '!drop':
+            matrix_api.send_message(event['room_id'], cmd_user_drop(event['sender']))
+
+        elif msg[0] == '!status':
+            matrix_api.send_message(event['room_id'], cmd_user_status(event['sender']))
+
+        elif msg[0] == '!join':
+            if len(msg) > 1:
+                matrix_api.send_message(event['room_id'], cmd_user_join(event['sender'], msg[1]))
+            else:
+                matrix_api.send_message(event['room_id'], cmd_user_join(event['sender']))
+
+
+    except Exception:
+        errmsg = "- on_direct_message -"
+        logger.exception(errmsg)
+
+    return jsonify({})
+
+def cmd_user_help(cmd=None):
+    reply = "This is an admin room for controlling your connection to pnut.io\n"
+    reply += "The following commands are available.\n\n"
+    reply += "!auth\t\t\t-  Authorize your account on pnut.io\n"
+    reply += "!save <token>\t-  Save your pnut.io auth token\n"
+    reply += "!drop\t\t\t-  Drop your pnut.io auth token\n"
+    reply += "!status\t\t\t-  Status of your pnut.io auth token\n"
+    reply += "!join <channel #>\t-  Join a private channel on pnut.io\n"
+
+    return reply
+
+def cmd_user_auth():
+    reply = "Visit the following URL to authorize your account on pnut.io.\n\n"
+    reply += "https://pnut.io/oauth/authenticate"
+    reply += "?client_id=6SeCRCpCZkmZOKFLFGWbcdAeq2fX1M5t"
+    reply += "&redirect_uri=urn:ietf:wg:oauth:2.0:oob"
+    reply += "&scope=write_post,presence,messages&response_type=token\n\n"
+    reply += "You will be provided with a token that you store with the !save command.\n"
+
+    return reply
+
+def cmd_user_save(sender=None, token=None):
+    if token is None:
+        reply = "You must provide a token with this command.\n"
+        reply += "!save <token>"
+        return reply
+
+    pnutpy.api.add_authorization_token(token)
+    try:
+        response, meta = pnutpy.api.get_user('me')
+
+        user = Users(
+            matrix_id=sender,
+            pnut_user_id=response.id,
+            pnut_user_token=token
+        )
+        db_session.add(user)
+        db_session.commit()
+
+        reply = "Success! You are now authorized as " + response['username']
+
+    except pnutpy.errors.PnutAuthAPIException as e:
+        reply = "Error!  Unable to authorize your account."
+
+    except Exception as e:
+        logging.exception('!save')
+        reply = "Error!  Problem saving your token."
+
+    return reply
+
+def cmd_user_drop(sender=None):
+    try:
+        user = Users.query.filter(Users.matrix_id == sender).one_or_none()
+        if user is not None:
+            db_session.delete(user)
+            db_session.commit()
+            reply = "Success!  Your auth token has been removed."
+        else:
+            reply = "You do not appear to be registered."
+
+    except Exception as e:
+        logging.exception('!drop')
+        reply = "Error!  Problem removing your token."
+
+    return reply
+
+def cmd_user_status(sender=None):
+    try:
+        user = Users.query.filter(Users.matrix_id == sender).one_or_none()
+        if user is None:
+            reply = "You are currently not authorized on pnut.io"
+        else:
+            pnutpy.api.add_authorization_token(user.pnut_user_token)
+            response, meta = pnutpy.api.get_user('me')
+            reply = "You are currently authorized as " + response.username
+
+    except pnutpy.errors.PnutAuthAPIException as e:
+        reply = "You are currently not authorized on pnut.io"
+
+    except Exception as e:
+        logging.exception('!status')
+        reply = "Error!  There was a problem checking your account."
+
+    return reply
+
+def cmd_user_join(sender=None, channel_id=None):
+    if channel_id is None:
+        reply = "You must provide a channel id number with this command.\n"
+        reply += "!join <channel #>"
+        return reply
+
+    try:
+        user = Users.query.filter(Users.matrix_id == sender).one_or_none()
+        if user is None:
+            reply = "You are currently not authorized on pnut.io"
+        else:
+            pnutpy.api.add_authorization_token(user.pnut_user_token)
+            channel, meta = pnutpy.api.get_channel(channel_id, include_raw=1)
+            room = Rooms.query.filter(Rooms.pnut_chan == channel_id).one_or_none()
+            if room is None:
+                create_room(channel, user)
+            else:
+                matrix_api = MatrixHttpApi(app.config['MATRIX_HOST'],
+                    token=app.config['MATRIX_AS_TOKEN'])
+                matrix_api.invite_user(room.room_id, sender)
+            reply = "ok"
+
+    except pnutpy.errors.PnutAuthAPIException as e:
+        reply = "You are currently not authorized on pnut.io"
+
+    except pnutpy.errors.PnutPermissionDenied:
+        reply = "You are not authorized for this channel"
+
+    except Exception:
+        logging.exception('!join')
+        reply = "Error!  There was a problem joining the channel."
+
+    return reply

appservice.yaml-sample

@@ -0,0 +1,13 @@
+id: "pnut"
+url: "http://127.0.0.1:5000"
+as_token: "<AS_AUTH_TOKEN>"
+hs_token: "<HS_AUTH_TOKEN>"
+sender_localpart: pnut
+namespaces:
+  users:
+    - exclusive: true
+      regex: "@pnut_.*"
+  rooms: []
+  aliases:
+    - exclusive: true
+      regex: "#pnut_.*"

config.yaml-sample

@@ -1,14 +1,37 @@
 SERVICE_DB: 'sqlite:///store.db'  # URL for the service database
 LISTEN_PORT: 5000  # matrix app service port to listen on
-MATRIX_HOST: 'https://localhost:8448'  # URL of the matrix server
+MATRIX_URL: 'https://<DOMAIN_NAME>' # public base URL of the matrix server
+MATRIX_HOST: 'https://localhost:8448'  # client URL of the matrix server
 MATRIX_DOMAIN: '<DOMAIN_NAME>'  # domain of the matrix server (right hand side of a matrix ID)
 MATRIX_AS_ID: '<MATRIX_ID>'  # matrix ID for the app service user
-MATRIX_AS_TOKEN: '<AUTH_TOKEN>'  # auth token for the app service user
-MATRIX_HS_TOKEN: '<AUTH_TOKEN>'  # auth token for the matrix server
-MATRIX_PNUT_PREFIX: '<APP_SERVICE_PREFIX>'  # prefix used for reserving matrix IDs and room aliases
+MATRIX_AS_TOKEN: '<AS_AUTH_TOKEN>'  # auth token for the app service user
+MATRIX_HS_TOKEN: '<HS_AUTH_TOKEN>'  # auth token for the matrix server
+MATRIX_PNUT_PREFIX: 'pnut_'  # prefix used for reserving matrix IDs and room aliases
 MATRIX_ADMIN_ROOM: '<ROOM ID>'  # Administrator control room ID
 MATRIX_PNUT_USER: '<USERNAME>'  # pnut.io username for the matrix bot
 MATRIX_PNUT_TOKEN: '<AUTH_TOKEN>'  # pnut.io auth token for the matrix bot
 PNUTCLIENT_ID: '<CLIENT_ID>'  # pnut.io app client ID
 PNUT_APPTOKEN: '<APP TOKEN>'  # pnut.io app token
 PNUT_APPKEY: '<APPSTREAM KEY>'  # pnut.io app stream key
+
+logging:
+  version: 1
+  formatters:
+    precise:
+      format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(message)s'
+    normal:
+      format: '%(name)s - %(levelname)s - %(message)s'
+  handlers:
+    console:
+      class: logging.StreamHandler
+      formatter: normal
+  loggers:
+    werkzeug:
+      level: DEBUG
+    appservice:
+      level: DEBUG
+    urllib3.connectionpool:
+      level: DEBUG
+  root:
+    level: DEBUG
+    handlers: [console]

database.py

@@ -2,9 +2,12 @@ from sqlalchemy import create_engine
 from sqlalchemy.orm import scoped_session, sessionmaker
 from sqlalchemy.ext.declarative import declarative_base
 import yaml
+import os
 
-with open("config.yaml", "rb") as config_file:
-    config = yaml.load(config_file)
+configyaml = os.environ.get("CONFIG_FILE")
+
+with open(configyaml, "rb") as config_file:
+    config = yaml.load(config_file, Loader=yaml.SafeLoader)
 
 engine = create_engine(config['SERVICE_DB'])
 db_session = scoped_session(sessionmaker(bind=engine))

models.py

@@ -14,6 +14,18 @@ class Rooms(Base):
     pnut_chan = Column(Integer, unique=True)
     portal = Column(Boolean)
 
+class DirectRooms(Base):
+    __tablename__ = 'direct'
+    id = Column(Integer, primary_key=True)
+    room_id = Column(String(250), unique=True)
+    pnut_chan = Column(Integer, unique=True)
+    bridge_user = Column(String(250))
+
+class ControlRooms(Base):
+    __tablename__ = 'control'
+    id = Column(Integer, primary_key=True)
+    room_id = Column(String(250), unique=True)
+
 class Events(Base):
     __tablename__ = 'events'
     id = Column(Integer, primary_key=True)

pnut-matrix.py

@@ -2,15 +2,19 @@ import websocket
 import threading
 import time
 import logging
+import logging.config
 import yaml
 import json
 import pnutpy
 import requests
 import magic
+import argparse
+import os
+import re
 
 from matrix_client.api import MatrixHttpApi
 from matrix_client.api import MatrixError, MatrixRequestError
-from models import Avatars, Rooms, Events
+from models import Avatars, Rooms, Events, DirectRooms, Users
 from database import db_session, init_db
 from sqlalchemy import and_
 from appservice import app
@@ -18,8 +22,24 @@ from appservice import app
 logger = logging.getLogger()
 
 _shutdown = threading.Event()
+_reconnect = threading.Event()
 
-def new_message(msg):
+class MLogFilter(logging.Filter):
+
+    ACCESS_TOKEN_RE = re.compile(r"(\?.*access(_|%5[Ff])token=)[^&]*(\s.*)$")
+    ACCESS_TOKEN_RE2 = re.compile(r"(\?.*access(_|%5[Ff])token=)[^&]*(.*)$")
+
+    def filter(self, record):
+        if record.name == "werkzeug" and len(record.args) > 0:
+            redacted_uri = MLogFilter.ACCESS_TOKEN_RE.sub(r"\1<redacted>\3", record.args[0])
+            record.args = (redacted_uri, ) + record.args[1:]
+        elif record.name == "urllib3.connectionpool" and len(record.args) > 3:
+            redacted_uri = MLogFilter.ACCESS_TOKEN_RE2.sub(r"\1<redacted>\3", record.args[4])
+            record.args = record.args[:4] + (redacted_uri,) + record.args[5:]
+
+        return True
+
+def new_message(msg, meta):
     logger.debug("channel: " + msg.channel_id)
     logger.debug("username: " + msg.user.username)
     if 'name' in msg.user:
@@ -33,14 +53,39 @@ def new_message(msg):
     if msg.source.id == config['PNUTCLIENT_ID']:
         return
 
-    room = Rooms.query.filter(Rooms.pnut_chan == msg.channel_id).one_or_none()
+    if meta['channel_type'] == 'io.pnut.core.chat':
+        room = Rooms.query.filter(Rooms.pnut_chan == msg.channel_id).one_or_none()
+    elif meta['channel_type'] == 'io.pnut.core.pm':
+        room = DirectRooms.query.filter(DirectRooms.pnut_chan == msg.channel_id).one_or_none()
+        if room is None:
+            # Do do an invite from the bridge user?
+            logger.debug('new invite?')
+            # create room and included matrix recpient
+            # subscribed_user_ids from meta
+            logger.debug(meta['subscribed_user_ids'])
+            pnut_user = matrix_id_from_pnut(msg.user.username)
+            profile = get_matrix_profile(pnut_user)
+            if not profile:
+                new_matrix_user(msg.user.username)
+            invitees=[]
+            for pm_user in meta['subscribed_user_ids']:
+                user = Users.query.filter(Users.pnut_user_id == pm_user).one_or_none()
+                if int(pm_user) == msg.user.id:
+                    continue
+                if user is not None:
+                    invitees.append(user.matrix_id)
+            if len(invitees) > 0:
+                room = new_room(pnut_user, invitees, msg.channel_id)
+    else:
+        room = None
+
     logger.debug(room)
     if room is None:
         logger.debug('-not_mapped-')
         return
 
     matrix_id = matrix_id_from_pnut(msg.user.username)
-    matrix_api = MatrixHttpApi(config['MATRIX_HOST'], 
+    matrix_api = MatrixHttpApi(config['MATRIX_HOST'],
         token=config['MATRIX_AS_TOKEN'],
         identity=matrix_id)
 
@@ -49,13 +94,13 @@ def new_message(msg):
         new_matrix_user(msg.user.username)
         logger.debug('-new_user-')
         profile = {'displayname': None}
-    
+
     if profile['displayname'] != matrix_display_from_pnut(msg.user):
         set_matrix_display(msg.user)
         logger.debug('-set_display-')
-    
+
     avatar = Avatars.query.filter(Avatars.pnut_user == msg.user.username).one_or_none()
-    if avatar is None or avatar.avatar != msg.user.content.avatar_image.link:
+    if avatar is None or avatar.avatar != msg.user.content.avatar_image.url:
         set_matrix_avatar(msg.user)
         logger.debug('-set_avatar-')
 
@@ -67,75 +112,84 @@ def new_message(msg):
 
     if 'content' in msg:
         text = msg.content.text + "\n"
-        ts = int(msg.created_at.strftime('%s')) * 1000
+        ts = int(time.time()) * 1000
 
         lnktext = ""
         for link in msg.content.entities.links:
             if 'title' in link:
                 lnktext += link.title + "\n"
-            if 'link' in link:
-                lnktext += link.link + "\n"
+            if 'url' in link:
+                lnktext += link.url + "\n"
 
         if len(lnktext) > 0:
             text += "\n" + lnktext
 
         r = matrix_api.send_message(room.room_id, text, timestamp=ts)
         event = Events(
-            event_id=r['event_id'], 
-            room_id=room.room_id, 
-            pnut_msg_id=msg.id, 
-            pnut_user_id=msg.user.id, 
+            event_id=r['event_id'],
+            room_id=room.room_id,
+            pnut_msg_id=msg.id,
+            pnut_user_id=msg.user.id,
             pnut_chan_id=msg.channel_id,
             deleted=False)
         db_session.add(event)
         db_session.commit()
 
-        if len(msg.raw) > 0:
+        if 'raw' in msg:
             logger.debug('-handle media uploads-')
             new_media(room.room_id, msg)
 
 def new_media(room_id, msg):
     matrix_id = matrix_id_from_pnut(msg.user.username)
-    matrix_api = MatrixHttpApi(config['MATRIX_HOST'], 
+    matrix_api = MatrixHttpApi(config['MATRIX_HOST'],
         token=config['MATRIX_AS_TOKEN'],
         identity=matrix_id)
-    ts = int(msg.created_at.strftime('%s')) * 1000
+    ts = int(time.time()) * 1000
 
-    for item in msg.raw:
-        if item.type == 'io.pnut.core.oembed' and 'url' in item.value:
+    if 'io.pnut.core.oembed' in msg.raw:
 
-            dl = requests.get(item.value.url, stream=True)
+        for oembed in msg.raw['io.pnut.core.oembed']:
+            info = {}
+
+            if oembed.type == 'photo':
+                msgtype = 'm.image'
+                dl_url = oembed.url
+                info['h'] = oembed.height
+                info['w'] = oembed.width
+            elif oembed.type == 'audio':
+                logger.debug("* recieved audio attachment")
+                continue
+            elif oembed.type == 'video':
+                logger.debug("* recieved video attachment")
+                continue
+            elif oembed.type == 'html5video':
+                logger.debug("* recieved html5 video attachment")
+                continue
+            elif oembed.type == 'rich':
+                logger.debug("* recieved video attachment")
+                continue
+            else:
+                logger.debug("* recieved unknown attachment")
+                continue
+
+            dl = requests.get(dl_url, stream=True)
             dl.raise_for_status()
             with magic.Magic(flags=magic.MAGIC_MIME_TYPE) as m:
-                mtype = m.id_buffer(dl.content)
-            info = {'mimetype': mtype}
+                info['mimetype'] = m.id_buffer(dl.content)
+            info['size'] = len(dl.content)
+            ul = matrix_api.media_upload(dl.content, info['mimetype'])
 
-            ul = matrix_api.media_upload(dl.content, mtype)
-
-            if item.value.type == 'photo':
-                msgtype = 'm.image'
-                info['h'] = item.value.height
-                info['w'] = item.value.width
-                info['size'] = len(dl.content)
-            elif item.value.type == 'video' or item.value.type == 'html5video':
-                msgtype = 'm.video'
-                info['h'] = item.value.height
-                info['w'] = item.value.width
-                info['size'] = len(dl.content)
-            elif item.value.type == 'audio':
-                msgtype = 'm.audio'
-                info['duration'] = int(item.value.duration) * 1000
-                info['size'] = len(dl.content)
+            if 'title' in oembed:
+                title = oembed.title
             else:
-                msgtype = 'm.file'
-                info['size'] = len(dl.content)
+                title = ""
 
-            r = matrix_api.send_content(room_id, ul['content_uri'], item.value.title, msgtype, extra_information=info, timestamp=ts)
+            r = matrix_api.send_content(room_id, ul['content_uri'], title, msgtype, extra_information=info, timestamp=ts)
             event = Events(
-                event_id=r['event_id'], 
-                room_id=room_id, 
-                pnut_msg_id=msg.id, 
-                pnut_user_id=msg.user.id, 
+                event_id=r['event_id'],
+                room_id=room_id,
+                pnut_msg_id=msg.id,
+                pnut_user_id=msg.user.id,
                 pnut_chan_id=msg.channel_id,
                 deleted=False)
             db_session.add(event)
@@ -143,7 +197,7 @@ def new_media(room_id, msg):
 
 def delete_message(msg):
     matrix_id = matrix_id_from_pnut(msg.user.username)
-    matrix_api = MatrixHttpApi(config['MATRIX_HOST'], 
+    matrix_api = MatrixHttpApi(config['MATRIX_HOST'],
         token=config['MATRIX_AS_TOKEN'],
         identity=matrix_id)
 
@@ -158,7 +212,7 @@ def matrix_id_from_pnut(username):
 
 def matrix_display_from_pnut(user):
     if 'name' in user:
-        display = user.name + " <@" + user.username + "> (pnut)"
+        display = user.name + " (@" + user.username + ")"
     else:
         display = "@" + user.username
     return display
@@ -175,18 +229,18 @@ def get_matrix_profile(matrix_id):
 
 def set_matrix_display(user):
     matrix_id = matrix_id_from_pnut(user.username)
-    matrix_api = MatrixHttpApi(config['MATRIX_HOST'], 
+    matrix_api = MatrixHttpApi(config['MATRIX_HOST'],
         token=config['MATRIX_AS_TOKEN'],
         identity=matrix_id)
     matrix_api.set_display_name(matrix_id, matrix_display_from_pnut(user))
 
 def set_matrix_avatar(user):
     matrix_id = matrix_id_from_pnut(user.username)
-    matrix_api = MatrixHttpApi(config['MATRIX_HOST'], 
+    matrix_api = MatrixHttpApi(config['MATRIX_HOST'],
         token=config['MATRIX_AS_TOKEN'],
         identity=matrix_id)
 
-    dl = requests.get(user.content.avatar_image.link, stream=True)
+    dl = requests.get(user.content.avatar_image.url, stream=True)
     dl.raise_for_status()
     with magic.Magic(flags=magic.MAGIC_MIME_TYPE) as m:
         mtype = m.id_buffer(dl.content)
@@ -194,32 +248,36 @@ def set_matrix_avatar(user):
 
     try:
         matrix_api.set_avatar_url(matrix_id, ul['content_uri'])
-        avatar = Avatars(pnut_user=user.username, avatar=user.content.avatar_image.link)
-        db_session.add(avatar)
+        avatar = Avatars.query.filter(Avatars.pnut_user == user.username).one_or_none()
+        if avatar is None:
+            avatar = Avatars(pnut_user=user.username, avatar=user.content.avatar_image.url)
+            db_session.add(avatar)
+        else:
+            avatar.avatar = user.content.avatar_image.url
         db_session.commit()
 
     except MatrixRequestError:
         logger.exception('failed to set user avatar')
 
 def new_matrix_user(username):
-    matrix_api = MatrixHttpApi(config['MATRIX_HOST'], 
+    matrix_api = MatrixHttpApi(config['MATRIX_HOST'],
         token=config['MATRIX_AS_TOKEN'])
     data = {
         'type': 'm.login.application_service',
-        'user': config['MATRIX_PNUT_PREFIX'] + username
+        'username': config['MATRIX_PNUT_PREFIX'] + username
     }
     matrix_api.register(content=data)
 
 def join_room(room_id, matrix_id):
-    matrix_api_as = MatrixHttpApi(config['MATRIX_HOST'], 
+    matrix_api_as = MatrixHttpApi(config['MATRIX_HOST'],
         token=config['MATRIX_AS_TOKEN'])
-    matrix_api = MatrixHttpApi(config['MATRIX_HOST'], 
+    matrix_api = MatrixHttpApi(config['MATRIX_HOST'],
         token=config['MATRIX_AS_TOKEN'],
         identity=matrix_id)
-    
+
     try:
         matrix_api.join_room(room_id)
-    
+
     except MatrixRequestError as e:
         if e.code == 403:
             matrix_api_as.invite_user(room_id, matrix_id)
@@ -229,63 +287,118 @@ def join_room(room_id, matrix_id):
 
     logger.debug('-room_join-')
 
+def new_room(pnut_user, invitees, chan):
+    dr = None
+    matrix_api = MatrixHttpApi(config['MATRIX_HOST'],
+        token=config['MATRIX_AS_TOKEN'],
+        identity=pnut_user)
+    url = matrix_url + '/createRoom'
+    params = {"access_token": config['MATRIX_AS_TOKEN'], "user_id": pnut_user}
+    content = {"visibility": "private", "is_direct": True, "invite": invitees}
+    headers = {"Content-Type": "application/json"}
+    r = requests.post(url, headers=headers, params=params,
+            data=json.dumps(content))
+    response = r.json()
+    for bridge_user in invitees:
+        dr = DirectRooms(room_id=response['room_id'],
+                bridge_user=pnut_user, pnut_chan=chan)
+        logger.debug(dr)
+        db_session.add(dr)
+        db_session.commit()
+
+    return dr
+
 def on_message(ws, message):
     # logger.debug("on_message: " + message)
     msg = json.loads(message)
     logger.debug(msg['meta'])
-    
+
     if 'data' in msg:
 
-        if msg['meta']['type'] == "message":
+        if 'channel_type' in msg['meta']:
 
-            # TODO: bypassed other channel types for now
-            if msg['meta']['channel_type'] != 'io.pnut.core.chat':
+            if msg['meta']['channel_type'] not in ['io.pnut.core.chat',
+                    'io.pnut.core.pm']:
                 return
 
-            pmsg = pnutpy.models.Message.from_response_data(msg['data'])
+            for d_item in msg['data']:
+                pmsg = pnutpy.models.Message.from_response_data(d_item)
 
-            if 'is_deleted' in msg['meta']:
-                if msg['meta']['is_deleted']:
-                    logger.debug("message: delete")
-                    delete_message(pmsg)
+                if 'is_deleted' in msg['meta']:
+                    if msg['meta']['is_deleted']:
+                        logger.debug("message: delete")
+                        delete_message(pmsg)
                 else:
-                    logger.debug("uh whut?")
-            else:
-                new_message(pmsg)
+                    new_message(pmsg, msg['meta'])
 
 def on_error(ws, error):
-    logger.debug("on_error: !!! ERROR !!!")
+    logger.error("on_error: !!! ERROR !!!")
     logger.error(error)
 
 def on_close(ws):
     logger.debug("on_close: ### CLOSED ###")
-    if not _shutdown.set():
-        time.sleep(10)
-        t.start()
-    else:
-        time.sleep(2)
 
 def on_open(ws):
 
     def run(*args):
-        while not _shutdown.isSet():
+        while not _shutdown.isSet() and not _reconnect.isSet():
             time.sleep(3)
-            ws.send(".")
+            try:
+                ws.send(".")
+            except websocket._exceptions.WebSocketConnectionClosedException:
+                logger.debug('websocket closed exception caught...')
+                _reconnect.set()
+
         time.sleep(1)
-        ws.close()
-        logger.debug("*** terminate ***")
-    
+        logger.debug("*** terminate thread ***")
+
     t = threading.Thread(target=run)
     t.start()
 
-if __name__ == '__main__':
-    # websocket.enableTrace(True)
-    logging.basicConfig(level=logging.INFO)
+def wsthreader(threadfunc):
 
-    with open("config.yaml", "rb") as config_file:
-        config = yaml.load(config_file)
-    
-    ws_url = 'wss://stream.pnut.io/v0/app?access_token=' 
+    def wrapper():
+
+        while not _shutdown.isSet():
+            _reconnect.clear()
+            logger.debug('threadfunc start...')
+            running = threadfunc()
+            logger.debug('threadfunc end...')
+            if running:
+                time.sleep(5)
+            else:
+                _shutdown.set()
+        logger.debug('*** thread stopped ***')
+
+    return wrapper
+
+if __name__ == '__main__':
+
+
+    a_parser = argparse.ArgumentParser()
+    a_parser.add_argument(
+        '-d', action='store_true', dest='debug',
+        help="debug logging"
+    )
+    # TODO: solve the database.py problem and enable this
+    # a_parser.add_argument(
+    #     '-c', '--config', default="config.yaml",
+    #     help="configuration file"
+    # )
+    args = a_parser.parse_args()
+
+    configyaml = os.environ.get("CONFIG_FILE")
+
+    with open(configyaml, "rb") as config_file:
+        config = yaml.load(config_file, Loader=yaml.SafeLoader)
+
+    # websocket.enableTrace(True)
+    logging.config.dictConfig(config['logging'])
+    redact_filter = MLogFilter()
+    logging.getLogger("werkzeug").addFilter(redact_filter)
+    logging.getLogger("urllib3.connectionpool").addFilter(redact_filter)
+
+    ws_url = 'wss://stream.pnut.io/v1/app?access_token='
     ws_url += config['PNUT_APPTOKEN'] + '&key=' + config['PNUT_APPKEY']
     ws_url += '&include_raw=1'
     matrix_url = config['MATRIX_HOST'] + '/_matrix/client/r0'
@@ -294,20 +407,20 @@ if __name__ == '__main__':
     init_db()
 
     # setup the websocket connection
-    ws = websocket.WebSocketApp(ws_url, on_message=on_message, 
+    ws = websocket.WebSocketApp(ws_url, on_message=on_message,
         on_error=on_error, on_close=on_close)
     ws.on_open = on_open
-    t = threading.Thread(target=ws.run_forever)
-    t.daemon = True
-    t.start()
+    wst = threading.Thread(target=wsthreader(ws.run_forever))
+    wst.start()
 
     # setup the matrix app service
     if config['MATRIX_ADMIN_ROOM']:
         logger.debug("- sould join admin room -")
         join_room(config['MATRIX_ADMIN_ROOM'], config['MATRIX_AS_ID'])
     app.config.update(config)
-    app.run(port=config['LISTEN_PORT'])
+    app.run(host=config['LISTEN_HOST'], port=config['LISTEN_PORT'])
 
-    _shutdown.set()
     logger.info('!! shutdown initiated !!')
+    _shutdown.set()
+    ws.close()
     time.sleep(2)

requirements.txt

@@ -1,9 +1,8 @@
 pyyaml
 requests
-matrix-client
+matrix-client==0.3.2
 Flask
 pnutpy
 sqlalchemy
 websocket-client
 filemagic
-git+https://github.com/shawnanastasio/python-matrix-bot-api

wait-for-synapse.sh

@@ -0,0 +1,26 @@
+#!/bin/sh
+# wait-for-synapse.sh
+#
+# based on an exmaple from https://docs.docker.com/compose/startup-order/
+#
+# command: ["/usr/src/app/wait-for-synapse.sh", "http://synapse:8008", "python", "/usr/src/app/pnut-matrix.py", "-d"]
+
+set -e
+
+host="$1"
+# Shift arguments with mapping:
+# - $0 => $0
+# - $1 => <discarded>
+# - $2 => $1
+# - $3 => $2
+# - ...
+# This is done for `exec "$@"` below to work correctly
+shift
+
+until [ "200" -eq $(curl -s -o /dev/null --head -w "%{http_code}" ${host}/_matrix/client/versions) ]; do
+    >&2 echo "synapse is unavailable - sleeping"
+    sleep 3
+done
+
+>&2 echo "synapse is up - executing command"
+exec "$@"