diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c42e8b..76397f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,9 +5,25 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] +## [1.0.0] - 2019-01-03 +### Fixed +- database initialization + +### Added +- Support for pnut app streams +- Sync avatars from pnut to matrix +- Administrator control room functions +- Example configuration file + +### Changed +- Display names for matrix users not registered with pnut +- Simplified storage models + ## 0.0.1 - 2018-08-23 ### Added - This CHANGELOG file because I can't believe for the last year I wasn't keeping track of releases for this project. :p -[Unreleased]: https://gitlab.dreamfall.space/thrrgilag/pnut-matrix/compare/v0.0.1...HEAD +[Unreleased]: https://gitlab.dreamfall.space/thrrgilag/pnut-matrix/compare/1.0.0...HEAD +[1.0.0]: https://gitlab.dreamfall.space/thrrgilag/pnut-matrix/tags/1.0.0 +[0.0.1]: https://gitlab.dreamfall.space/thrrgilag/pnut-matrix/tags/v0.0.1 diff --git a/appservice.py b/appservice.py index 60d0698..46fc05a 100644 --- a/appservice.py +++ b/appservice.py @@ -4,238 +4,251 @@ import logging import re import pnutpy -from bot import MonkeyBot -# from pnutlib import Pnut +from matrix_client.api import MatrixHttpApi +from matrix_client.api import MatrixError, MatrixRequestError +from models import Avatars, Rooms, Events, Users +from database import db_session +from sqlalchemy import and_ from flask import Flask, jsonify, request, abort -from models import * + +logger = logging.getLogger(__name__) app = Flask(__name__) -app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False -db.init_app(app) -cmdbot = MonkeyBot() +@app.errorhandler(404) +def not_found(error): + return jsonify({'errcode':'COM.MONKEYSTEW.PNUT_NOT_FOUND'}), 404 -txId = 0 +@app.errorhandler(403) +def forbidden(error): + return jsonify({'errcode':'COM.MONKEYSTEW.PNUT_FORBIDDEN'}), 403 -@app.route("/transactions/", methods=["PUT"]) -def on_receive_events(transaction): - global txId - - events = request.get_json()["events"] - for event in events: - # logging.info(event) - user = MatrixUser.query.filter_by(matrix_id=event["user_id"]).first() - - if (event['type'] == 'm.room.message' - and not app.config['MATRIX_PNUT_PREFIX'] in event["user_id"] - and not 'pnut-bridge' in event["user_id"]): - - embed = None - - chan = MatrixRoom2.query.filter_by(room_id=event['room_id']).first() - if chan: - chan_id = chan.pnut_chan - else: - adminrm = MatrixAdminRooms.query.filter_by(room_id=event['room_id']).first() - if adminrm: - cmdbot.on_message(event) - return jsonify({}) - - if 'msgtype' not in event['content']: - return jsonify({}) - - if (event['content']['msgtype'] == 'm.text' - or event['content']['msgtype'] == 'm.notice'): - if user: - token = user.pnut_token - text = event['content']['body'] - else: - token = app.config['MATRIX_PNUT_TOKEN'] - text = "[" + get_displayname(event["user_id"]) + "] " + event['content']['body'] - elif event['content']['msgtype'] == 'm.emote': - if user: - token = user.pnut_token - text = "* " + user.pnut_id + " " + event['content']['body'] - else: - token = app.config['MATRIX_PNUT_TOKEN'] - text = "* " + get_displayname(event["user_id"]) + " " + event['content']['body'] - elif event['content']['msgtype'] == 'm.image': - imgurl = app.config['MATRIX_HOST'] + '/_matrix/media/r0/download/' + event['content']['url'][6:] - - value = {"type": "photo", "version": "1.0"} - value["title"] = event['content']['body'] - value["url"] = imgurl - if 'info' in event['content']: - if 'w' in event['content']['info']: - value["width"] = event['content']['info']['w'] - else: - value["width"] = 200 - if 'h' in event['content']['info']: - value["height"] = event['content']['info']['h'] - else: - value["height"] = 200 - if 'thumbnail_info' in event['content']['info']: - thmburl = app.config['MATRIX_HOST'] + '/_matrix/media/r0/download/' + event['content']['info']['thumbnail_url'][6:] - value["thumbnail_width"] = event['content']['info']['thumbnail_info']['w'] - value["thumbnail_height"] = event['content']['info']['thumbnail_info']['h'] - value["thumbnail_url"] = thmburl - else: - if 'w' in event['content']['info']: - value["thumbnail_width"] = event['content']['info']['w'] - else: - value["thumbnail_width"] = 200 - if 'h' in event['content']['info']: - value["thumbnail_height"] = event['content']['info']['h'] - else: - value["thumbnail_height"] = 200 - value["thumbnail_url"] = imgurl - rawitem = {"type": "io.pnut.core.oembed", "value": value} - embed = [rawitem] - if user: - token = user.pnut_token - text = "" - else: - token = app.config['MATRIX_PNUT_TOKEN'] - text = "[" + get_displayname(event["user_id"]) + "] " - text += imgurl - else: - text = None - - if user == None and not chan.pnut_write: - txId += 1 - url = app.config['MATRIX_HOST'] - url += '/_matrix/client/r0/rooms/{0}/redact/{1}/{2}'.format( - event['room_id'], event['event_id'], txId) - url += "?access_token=" + app.config['MATRIX_AS_TOKEN'] - requests.put(url, headers={"Content-Type": "application/json"}, data=json.dumps({})) - else: - if text: - pnutpy.api.add_authorization_token(token) - try: - msg, meta = pnutpy.api.create_message(chan_id, data={'text': text, 'raw': embed}) - - if token == app.config['MATRIX_PNUT_TOKEN']: - puser = app.config['MATRIX_PNUT_USER'] - else: - puser = user.pnut_id - cctag = re.search('##$', text) - if cctag: - cname = get_channel_settings(chan_id)['name'] - text = text[:-2] - text += '\n\n[' + cname + "](https://patter.chat/room.html?channel=" + chan_id + ")" - r, meta = pnutpy.api.create_post(data={'text': text}) - - el = MatrixMsgEvents(event['event_id'], event['room_id'], msg.id, puser, chan_id) - db.session.add(el) - db.session.commit() - - except pnutpy.errors.PnutAuthAPIException: - txId += 1 - err_notice = event["user_id"] + ": The pnut.io channel this room is connected with is restricted to valid " - err_notice += "pnut.io users only. Start a direct chat to link your account." - body = {'msgtype': 'm.notice', 'body': err_notice} - url = app.config['MATRIX_HOST'] - url += '/_matrix/client/r0/rooms/{0}/send/m.room.message/{1}'.format(event['room_id'], str(txId)) - url += "?access_token=" + app.config['MATRIX_AS_TOKEN'] - requests.put(url, headers={"Content-Type": "application/json"}, data=json.dumps(body)) - - except: - logging.debug('*** an error occured while posting a message ***') - - - elif event['type'] == 'm.room.redaction': - r_event = MatrixMsgEvents.query.filter_by(event_id=event['redacts'],room_id=event['room_id'],deleted=False).first() - if r_event: - if r_event.pnut_user == app.config['MATRIX_PNUT_USER']: - token = app.config['MATRIX_PNUT_TOKEN'] - else: - r_user = MatrixUser.query.filter_by(pnut_id=r_event.pnut_user).first() - if r_user: - token = r_user.pnut_token - else: - return jsonify({}) - - r_event.deleted = True - db.session.commit() - pnutpy.api.add_authorization_token(token) - r, meta = pnutpy.api.delete_message(r_event.pnut_chan, r_event.pnut_msgid) - - elif event['type'] == 'm.room.member': - - if event['state_key'] == app.config['MATRIX_AS_ID']: - - if event['content']['membership'] == 'invite' and 'is_direct' in event['content'] and event['content']['is_direct'] == True: - - logging.info('>> GOT PRIVATE INVITE') - - - elif event['content']['membership'] == 'invite': - logging.info('>> GOT ROOM INVITE') - - - elif event['content']['membership'] == 'leave': - adminrm = MatrixAdminRooms.query.filter_by(room_id=event['room_id']).first() - - - - return jsonify({}) +@app.teardown_appcontext +def shutdown_session(exception=None): + db_session.remove() @app.route("/rooms/") def query_alias(alias): alias_localpart = alias.split(":")[0][1:] channel_id = int(alias_localpart.split('_')[1]) - # prevent room from being created if channel is already plumbed - chroom = MatrixRoom2.query.filter_by(pnut_chan=channel_id).first() - if chroom: + room = Rooms.query.filter(Rooms.pnut_chan == channel_id).one_or_none() + if room is not None: abort(404) + token = app.config['MATRIX_PNUT_TOKEN'] + pnutpy.api.add_authorization_token(token) try: - r = requests.get('https://api.pnut.io/v0/channels/' + str(channel_id) + '?include_raw=1') - if r.status_code == 200: - cdata = json.loads(r.text)['data'] - if cdata['type'] != 'io.pnut.core.chat': - abort(404) - if 'is_active' in cdata: - if not cdata['is_active']: - abort(404) - raw = cdata['raw'] - for item in raw: - if item['type'] == 'io.pnut.core.chat-settings': - chan_settings = item['value'] - else: + channel, meta = pnutpy.api.get_channel(channel_id, include_raw=1) + + if 'is_active' in channel and channel.is_active == False: + logger.debug("-channel isn't active-") abort(404) - except: - raise + + channel_settings = {} + for item in channel.raw: + if item.type == 'io.pnut.core.chat-settings': + channel_settings = item.value + + # Matrix sdk doesn't include all details in a single call + room = {'room_alias_name': alias_localpart} + if 'name' in channel_settings: + room['name'] = channel_settings['name'] + if 'description' in channel_settings: + room['topic'] = channel_settings['description'] + if channel.acl.read.any_user: + room['preset'] = 'public_chat' + room['visibility'] = 'public' + + url = app.config['MATRIX_HOST'] + '/_matrix/client/api/v1/createRoom?access_token=' + url += app.config['MATRIX_AS_TOKEN'] + headers = {"Content-Type":"application/json"} + r = requests.post(url, headers=headers, data=json.dumps(room)) + if r.status_code == 200: + pnutpy.api.subscribe_channel(channel_id) + rdata = r.json() + rr = Rooms( + room_id=rdata['room_id'], + pnut_chan=channel_id, + portal=True + ) + db_session.add(rr) + db_session.commit() + + except Exception: + logger.exception("-couldn't get the pnut channel-") abort(404) - mroom = {'room_alias_name': alias_localpart} - if 'name' in chan_settings: - mroom['name'] = chan_settings['name'] - if 'description' in chan_settings: - mroom['topic'] = chan_settings['description'] - if cdata['acl']['read']['any_user']: - mroom['preset'] = 'public_chat' - mroom['visibility'] = 'public' - - resp = requests.post( - # NB: "TOKEN" is the as_token referred to in registration.yaml - "http://localhost:8008/_matrix/client/api/v1/createRoom?access_token=" + app.config['MATRIX_AS_TOKEN'], - json.dumps(mroom), - headers={"Content-Type":"application/json"} - ) - - if resp.status_code == 200: - room_id = json.loads(resp.text)['room_id'] - mro = MatrixRoom2(room_id, channel_id, cdata['acl']['write']['any_user']) - db.session.add(mro) - db.session.commit() - return jsonify({}) -@app.errorhandler(404) -def not_found(error): - return jsonify({'errcode':'COM.MONKEYSTEW.PNUT_NOT_FOUND'}), 404 +@app.route("/transactions/", methods=["PUT"]) +def on_receive_events(transaction): + + access_token = request.args.get('access_token', '') + if access_token != app.config['MATRIX_HS_TOKEN']: + abort(403) + + events = request.get_json()["events"] + for event in events: + logger.debug(event) + + # TODO: route event if it's in the control room + if app.config['MATRIX_ADMIN_ROOM'] and app.config['MATRIX_ADMIN_ROOM'] == event['room_id']: + return on_admin_event(event) + + user = Users.query.filter(Users.matrix_id == event['user_id']).one_or_none() + + if event['type'] == 'm.room.message': + new_message(event, user) + + elif event['type'] == 'm.room.redaction': + delete_message(event, user) + + elif event['type'] == 'm.room.member': + logger.debug("-room member event") + + return jsonify({}) + +def new_message(event, user): + + if app.config['MATRIX_PNUT_PREFIX'] in event['user_id'] or 'pnut-bridge' in event['user_id']: + logger.debug('-skipping dup event-') + return + + room = Rooms.query.filter(Rooms.room_id == event['room_id']).one_or_none() + if room is None: + logger.debug('-room not mapped-') + return + + if 'msgtype' not in event['content']: + logger.debug('-unknown message type-') + return + + if user is not None: + token = user.pnut_user_token + prefix = "" + else: + token = app.config['MATRIX_PNUT_TOKEN'] + prefix = "[" + get_displayname(event['user_id']) + "] (" + event['user_id'] + ")\n" + prefix = prefix.replace('@', '@\v') + + pnutpy.api.add_authorization_token(token) + text = None + embed = None + + if event['content']['msgtype'] == 'm.text' or event['content']['msgtype'] == 'm.notice': + text = event['content']['body'] + + elif event['content']['msgtype'] == 'm.emote': + text = "* " + event['content']['body'] + + elif event['content']['msgtype'] == 'm.image': + text = event['content']['body'] + "\n" + text += app.config['MATRIX_HOST'] + '/_matrix/media/r0/download/' + event['content']['url'][6:] + embed = [raw_from_event(event)] + + elif event['content']['msgtype'] == 'm.video': + text = event['content']['body'] + "\n" + text += app.config['MATRIX_HOST'] + '/_matrix/media/r0/download/' + event['content']['url'][6:] + + elif event['content']['msgtype'] == 'm.audio': + text = event['content']['body'] + "\n" + text += app.config['MATRIX_HOST'] + '/_matrix/media/r0/download/' + event['content']['url'][6:] + + elif event['content']['msgtype'] == 'm.file': + text = event['content']['body'] + "\n" + text += app.config['MATRIX_HOST'] + '/_matrix/media/r0/download/' + event['content']['url'][6:] + + else: + logger.debug('-unknown msg type- ' + event['content']['msgtype']) + return + + text = prefix + text + try: + msg, meta = pnutpy.api.create_message(room.pnut_chan, data={'text': text, 'raw': embed}) + revent = Events( + event_id=event['event_id'], + room_id=event['room_id'], + pnut_msg_id=msg.id, + pnut_user_id=msg.user.id, + pnut_chan_id=room.pnut_chan, + deleted=False + ) + db_session.add(revent) + db_session.commit() + + if user is not None: + cctag = re.search('##$', text) + if cctag: + cname = get_channel_settings(room.pnut_chan)['name'] + text = text[:-2] + text += '\n\n[' + cname + "](https://patter.chat/room.html?channel=" + str(room.pnut_chan) + ")" + r, meta = pnutpy.api.create_post(data={'text': text}) + + except pnutpy.errors.PnutAuthAPIException: + logger.exception('-unable to post to pnut channel-') + return + + except Exception: + logger.exception('-something bad happened here-') + return + +def raw_from_event(event): + + url = app.config['MATRIX_HOST'] + '/_matrix/media/r0/download/' + event['content']['url'][6:] + + if event['content']['msgtype'] == 'm.image': + value = {'type': "photo", 'version': "1.0"} + value['url'] = url + value['title'] = event['content']['body'] + if 'info' in event['content']: + value['width'] = event['content']['info']['w'] + value['height'] = event['content']['info']['h'] + if 'thumbnail_info' in event['content']['info']: + value['thumbnail_url'] = app.config['MATRIX_HOST'] + '/_matrix/media/r0/download/' + event['content']['info']['thumbnail_url'][6:] + value['thumbnail_width'] = event['content']['info']['thumbnail_info']['w'] + value['thumbnail_height'] = event['content']['info']['thumbnail_info']['h'] + + elif event['content']['msgtype'] == 'm.video': + # TODO: Need to sort out the oembed for this media type + value = {'type': "html5video", 'version': "1.0"} + source = {'url': url} + value['title'] = event['content']['body'] + if 'info' in event['content']: + value['width'] = event['content']['info']['w'] + value['height'] = event['content']['info']['h'] + source['type'] = event['content']['info']['mimetype'] + else: + return None + value['sources'] = [source] + + elif event['content']['msgtype'] == 'm.audio': + # TODO: Need to sort out the oembed for this media type + value = {'type': "audio", 'version': "1.0"} + return None + else: + return None + + return {'type': "io.pnut.core.oembed", 'value': value} + +def delete_message(event, user): + + # TODO: should there be moderator handled redactions? + + if user is not None: + token = user.pnut_user_token + else: + token = app.config['MATRIX_PNUT_TOKEN'] + pnutpy.api.add_authorization_token(token) + + e = Events.query.filter(and_(Events.event_id == event['redacts'], Events.deleted == False)).one_or_none() + if e is None: + logger.debug("- can't find the event to remove -") + return + + r, meta = pnutpy.api.delete_message(e.pnut_chan_id, e.pnut_msg_id) + e.deleted = True + db_session.commit() def get_displayname(userid): url = "http://localhost:8008/_matrix/client/r0/profile/" + userid @@ -247,13 +260,175 @@ def get_displayname(userid): return userid def get_channel_settings(channel_id): - chan_settings = {} - r = requests.get('https://api.pnut.io/v0/channels/' + str(channel_id) + '?include_raw=1') - if r.status_code == 200: - cdata = r.json()['data'] - raw = cdata['raw'] - for item in raw: - if item['type'] == 'io.pnut.core.chat-settings': - chan_settings = item['value'] + channel_settings = {} + try: + channel, meta = pnutpy.api.get_channel(channel_id, include_raw=1) - return chan_settings + for item in channel.raw: + if item.type == 'io.pnut.core.chat-settings': + channel_settings = item.value + + except Exception: + logger.exception('-unable to get channel settings-') + + return channel_settings + +def on_admin_event(event): + matrix_api = MatrixHttpApi(app.config['MATRIX_HOST'], + token=app.config['MATRIX_AS_TOKEN']) + logger.debug("- admin room event recieved -") + + if event['type'] != 'm.room.message': + return jsonify({}) + + msg = event['content']['body'].split(' ') + + if msg[0] == 'help': + if len(msg) > 1: + matrix_api.send_message(event['room_id'], cmd_admin_help(msg[1])) + else: + matrix_api.send_message(event['room_id'], cmd_admin_help()) + + elif msg[0] == 'list': + matrix_api.send_message(event['room_id'], cmd_admin_list()) + + elif msg[0] == 'unlink': + if len(msg) > 1: + matrix_api.send_message(event['room_id'], cmd_admin_unlink(msg[1])) + else: + matrix_api.send_message(event['room_id'], cmd_admin_help('unlink')) + + elif msg[0] == 'link': + if len(msg) > 2: + matrix_api.send_message(event['room_id'], cmd_admin_link(msg[1], msg[2])) + else: + matrix_api.send_message(event['room_id'], cmd_admin_help('link')) + + return jsonify({}) + +def cmd_admin_help(cmd=None): + + help_usage = "help [command]" + help_desc = "Show information about available commands." + list_usage = "list" + list_desc = "List the rooms currently linked with pnut.io." + unlink_usage = "unlink | " + unlink_desc = "Unlink a room between Matrix and pnut.io." + link_usage = "link " + link_desc = "Link a room between Matrix and pnut.io." + + if cmd == 'help': + text = "usage: " + help_usage + "\n\n" + text += help_desc + if cmd == 'list': + text = "usage: " + list_usage + "\n\n" + text += list_desc + elif cmd == 'unlink': + text = "usage: " + unlink_usage + "\n\n" + text += unlink_desc + elif cmd == 'link': + text = "usage: " + link_usage + "\n\n" + text += link_desc + else: + text = "The following commands are available:\n\n" + text += help_usage + "\n" + text += list_usage + "\n" + text += unlink_usage + "\n" + text += link_usage + "\n" + + return text + +def cmd_admin_list(): + text = "" + rooms = Rooms.query.all() + + if len(rooms) > 0: + text = "ID\tMATRIX ID\tPNUT CHANNEL\n" + else: + text = " - no rooms are currently linked - \n" + + for room in rooms: + text += str(room.id) + '\t' + text += room.room_id + '\t\t\t\t\t' + text += str(room.pnut_chan) + '\t' + if room.portal: + text += "(portal)" + text += '\n' + + return text + +def cmd_admin_link(room_id, pnut_chan_id): + matrix_api = MatrixHttpApi(app.config['MATRIX_HOST'], + token=app.config['MATRIX_AS_TOKEN']) + pnutpy.api.add_authorization_token(app.config['MATRIX_PNUT_TOKEN']) + + mrcheck = Rooms.query.filter(Rooms.room_id == room_id).one_or_none() + pncheck = Rooms.query.filter(Rooms.pnut_chan == pnut_chan_id).one_or_none() + + if mrcheck is not None or pncheck is not None: + return "- room may already be linked -" + + try: + channel, meta = pnutpy.api.subscribe_channel(pnut_chan_id) + r = matrix_api.join_room(room_id) + + rec = Rooms( + room_id=room_id, + pnut_chan=channel.id, + portal=False + ) + db_session.add(rec) + db_session.commit() + + except pnutpy.errors.PnutAuthAPIException: + errmsg = "- unable to subscribe to channel -" + logger.exception(errmsg) + return errmsg + + except Exception: + errmsg = "- unable to link room for some reason -" + logger.exception(errmsg) + return errmsg + +def cmd_admin_unlink(id): + matrix_api = MatrixHttpApi(app.config['MATRIX_HOST'], + token=app.config['MATRIX_AS_TOKEN']) + pnutpy.api.add_authorization_token(app.config['MATRIX_PNUT_TOKEN']) + + if id.startswith('!'): + room = Rooms.query.filter(Rooms.room_id == id).one_or_none() + else: + room = Rooms.query.filter(Rooms.pnut_chan == id).one_or_none() + + if room.portal: + alias = "#" + app.config['MATRIX_PNUT_PREFIX'] + alias += str(room.pnut_chan) + ":" + alias += app.config['MATRIX_DOMAIN'] + matrix_api.remove_room_alias(alias) + + # Kicking users needs at least moderator privs + members = matrix_api.get_room_members(room.room_id) + reason = "Portal room has been unlinked by administrator" + for m in members['chunk']: + if m['membership'] == 'join' and m['sender'] != app.config['MATRIX_AS_ID']: + if room.portal: + matrix_api.kick_user(room.room_id, m['sender'], reason=reason) + else: + if m['sender'].startswith("@" + app.config['MATRIX_PNUT_PREFIX']): + matrix_api.kick_user(room.room_id, m['sender'], reason=reason) + + try: + channel, meta = pnutpy.api.unsubscribe_channel(room.pnut_chan) + matrix_api.leave_room(room.room_id) + + if room is not None: + db_session.delete(room) + db_session.commit() + return "- room has been unlinked -" + else: + return "- unable to locate room to unlink -" + + except Exception: + errmsg = "- error while unlinking room -" + logger.exception(errmsg) + return errmsg diff --git a/bot.py b/bot.py deleted file mode 100644 index a252938..0000000 --- a/bot.py +++ /dev/null @@ -1,93 +0,0 @@ -import requests -import logging -import yaml -import sys -import time -import shlex -import json -import re -import pnutpy - -from matrix_client.client import MatrixClient -from matrix_client.api import MatrixHttpApi -from matrix_client.api import MatrixError, MatrixRequestError -from models import * - -class MonkeyBot: - - txId = 0 - - def __init__(self): - with open("config.yaml", "rb") as config_file: - self.config = yaml.load(config_file) - self.api = MatrixHttpApi(self.config['MATRIX_HOST'], self.config['MATRIX_AS_TOKEN']) - self.pnut_token = self.config['MATRIX_PNUT_TOKEN'] - pnutpy.api.add_authorization_token(self.pnut_token) - - def on_invite(self, event): - logging.debug("<__on_invite__>") - logging.debug(event) - room = self.api.join_room(event['room_id']) - - def on_message(self, event): - logging.debug("<__on_message__>") - logging.debug(event) - - if event['type'] == 'm.room.message': - if event['content']['msgtype'] == 'm.text': - argv = shlex.split(event['content']['body']) - cmd = argv[0] - args = argv[1:] - self._parse_cmd(event, cmd, args) - - def _parse_cmd(self, event, cmd, args): - logging.debug("<__parse_cmd__>") - logging.debug(" " + cmd) - logging.debug(args) - - if cmd.lower() == 'help': - self.api.send_notice(event['room_id'], self._help()) - - elif cmd.lower() == 'set_access_token': - token = args[0] - pnutpy.api.add_authorization_token(token) - try: - response, meta = pnutpy.api.get_user('me') - - user = MatrixUser(matrix_id=event['user_id'], room_id=event['room_id'], - pnut_id=response['username'], pnut_token=token) - db.session.add(user) - db.session.commit() - reply = "Token verified, you are now linked as " + response['username'] - - except pnut.api.PnutAuthAPIException as e: - reply = "Your account is not authorized." - - except Exception as e: - reply = "Something went wrong...\n" - reply += str(e) - logging.exception('::set_access_token::') - - self.api.send_notice(event['room_id'], reply) - pnutpy.api.add_authorization_token(self.pnut_token) - - elif cmd.lower() == 'drop_access_token': - user = MatrixUser.query.filter_by(matrix_id=event['user_id']).first() - db.session.delete(user) - db.session.commit() - reply = "Your token has been removed." - self.api.send_notice(event['room_id'], reply) - - else: - self.api.send_notice(event['room_id'], self._help()) - - def _help(self): - reply = "Visit the following URL to authorize pnut-matrix with your account on pnut.io.\n\n" - reply += "https://pnut.io/oauth/authenticate?client_id=6SeCRCpCZkmZOKFLFGWbcdAeq2fX1M5t&redirect_uri=urn:ietf:wg:oauth:2.0:oob&scope=write_post,presence,messages&response_type=token\n\n" - reply += "The following commands are available.\n\n" - reply += "set_access_token \n" - reply += " - Set your access token for matrix -> pnut.io account puppeting\n\n" - reply += "drop_access_token\n" - reply += " - Drop your access token to remove puppeting\n\n" - return reply - diff --git a/config.yaml-sample b/config.yaml-sample new file mode 100644 index 0000000..18716e5 --- /dev/null +++ b/config.yaml-sample @@ -0,0 +1,14 @@ +SERVICE_DB: 'sqlite:///store.db' # URL for the service database +LISTEN_PORT: 5000 # matrix app service port to listen on +MATRIX_HOST: 'https://localhost:8448' # URL of the matrix server +MATRIX_DOMAIN: '' # domain of the matrix server (right hand side of a matrix ID) +MATRIX_AS_ID: '' # matrix ID for the app service user +MATRIX_AS_TOKEN: '' # auth token for the app service user +MATRIX_HS_TOKEN: '' # auth token for the matrix server +MATRIX_PNUT_PREFIX: '' # prefix used for reserving matrix IDs and room aliases +MATRIX_ADMIN_ROOM: '' # Administrator control room ID +MATRIX_PNUT_USER: '' # pnut.io username for the matrix bot +MATRIX_PNUT_TOKEN: '' # pnut.io auth token for the matrix bot +PNUTCLIENT_ID: '' # pnut.io app client ID +PNUT_APPTOKEN: '' # pnut.io app token +PNUT_APPKEY: '' # pnut.io app stream key diff --git a/database.py b/database.py new file mode 100644 index 0000000..c2f9e22 --- /dev/null +++ b/database.py @@ -0,0 +1,21 @@ +from sqlalchemy import create_engine +from sqlalchemy.orm import scoped_session, sessionmaker +from sqlalchemy.ext.declarative import declarative_base +import yaml + +with open("config.yaml", "rb") as config_file: + config = yaml.load(config_file) + +engine = create_engine(config['SERVICE_DB']) +db_session = scoped_session(sessionmaker(bind=engine)) + +Base = declarative_base() + +Base.query = db_session.query_property() + +def init_db(): + # import all modules here that might define models so that + # they will be registered properly on the metadata. Otherwise + # you will have to import them first before calling init_db() + import models + Base.metadata.create_all(bind=engine) diff --git a/manage.py b/manage.py deleted file mode 100644 index 12a3128..0000000 --- a/manage.py +++ /dev/null @@ -1,17 +0,0 @@ -import yaml -from appservice import app -from models import * -from flask_script import Manager -from flask_migrate import Migrate, MigrateCommand - -migrate = Migrate(app, db) - -manager = Manager(app) -manager.add_command('db', MigrateCommand) - -if __name__ == '__main__': - with open("config.yaml", "rb") as config_file: - config = yaml.load(config_file) - - app.config['SQLALCHEMY_DATABASE_URI'] = config['SQLALCHEMY_DATABASE_URI'] - manager.run() diff --git a/migrations/README b/migrations/README deleted file mode 100644 index 98e4f9c..0000000 --- a/migrations/README +++ /dev/null @@ -1 +0,0 @@ -Generic single-database configuration. \ No newline at end of file diff --git a/migrations/alembic.ini b/migrations/alembic.ini deleted file mode 100644 index f8ed480..0000000 --- a/migrations/alembic.ini +++ /dev/null @@ -1,45 +0,0 @@ -# A generic, single database configuration. - -[alembic] -# template used to generate migration files -# file_template = %%(rev)s_%%(slug)s - -# set to 'true' to run the environment during -# the 'revision' command, regardless of autogenerate -# revision_environment = false - - -# Logging configuration -[loggers] -keys = root,sqlalchemy,alembic - -[handlers] -keys = console - -[formatters] -keys = generic - -[logger_root] -level = WARN -handlers = console -qualname = - -[logger_sqlalchemy] -level = WARN -handlers = -qualname = sqlalchemy.engine - -[logger_alembic] -level = INFO -handlers = -qualname = alembic - -[handler_console] -class = StreamHandler -args = (sys.stderr,) -level = NOTSET -formatter = generic - -[formatter_generic] -format = %(levelname)-5.5s [%(name)s] %(message)s -datefmt = %H:%M:%S diff --git a/migrations/env.py b/migrations/env.py deleted file mode 100644 index 4593816..0000000 --- a/migrations/env.py +++ /dev/null @@ -1,87 +0,0 @@ -from __future__ import with_statement -from alembic import context -from sqlalchemy import engine_from_config, pool -from logging.config import fileConfig -import logging - -# this is the Alembic Config object, which provides -# access to the values within the .ini file in use. -config = context.config - -# Interpret the config file for Python logging. -# This line sets up loggers basically. -fileConfig(config.config_file_name) -logger = logging.getLogger('alembic.env') - -# add your model's MetaData object here -# for 'autogenerate' support -# from myapp import mymodel -# target_metadata = mymodel.Base.metadata -from flask import current_app -config.set_main_option('sqlalchemy.url', - current_app.config.get('SQLALCHEMY_DATABASE_URI')) -target_metadata = current_app.extensions['migrate'].db.metadata - -# other values from the config, defined by the needs of env.py, -# can be acquired: -# my_important_option = config.get_main_option("my_important_option") -# ... etc. - - -def run_migrations_offline(): - """Run migrations in 'offline' mode. - - This configures the context with just a URL - and not an Engine, though an Engine is acceptable - here as well. By skipping the Engine creation - we don't even need a DBAPI to be available. - - Calls to context.execute() here emit the given string to the - script output. - - """ - url = config.get_main_option("sqlalchemy.url") - context.configure(url=url) - - with context.begin_transaction(): - context.run_migrations() - - -def run_migrations_online(): - """Run migrations in 'online' mode. - - In this scenario we need to create an Engine - and associate a connection with the context. - - """ - - # this callback is used to prevent an auto-migration from being generated - # when there are no changes to the schema - # reference: http://alembic.readthedocs.org/en/latest/cookbook.html - def process_revision_directives(context, revision, directives): - if getattr(config.cmd_opts, 'autogenerate', False): - script = directives[0] - if script.upgrade_ops.is_empty(): - directives[:] = [] - logger.info('No changes in schema detected.') - - engine = engine_from_config(config.get_section(config.config_ini_section), - prefix='sqlalchemy.', - poolclass=pool.NullPool) - - connection = engine.connect() - context.configure(connection=connection, - target_metadata=target_metadata, - process_revision_directives=process_revision_directives, - **current_app.extensions['migrate'].configure_args) - - try: - with context.begin_transaction(): - context.run_migrations() - finally: - connection.close() - -if context.is_offline_mode(): - run_migrations_offline() -else: - run_migrations_online() diff --git a/migrations/script.py.mako b/migrations/script.py.mako deleted file mode 100644 index 2c01563..0000000 --- a/migrations/script.py.mako +++ /dev/null @@ -1,24 +0,0 @@ -"""${message} - -Revision ID: ${up_revision} -Revises: ${down_revision | comma,n} -Create Date: ${create_date} - -""" -from alembic import op -import sqlalchemy as sa -${imports if imports else ""} - -# revision identifiers, used by Alembic. -revision = ${repr(up_revision)} -down_revision = ${repr(down_revision)} -branch_labels = ${repr(branch_labels)} -depends_on = ${repr(depends_on)} - - -def upgrade(): - ${upgrades if upgrades else "pass"} - - -def downgrade(): - ${downgrades if downgrades else "pass"} diff --git a/migrations/versions/0ddf19141ead_.py b/migrations/versions/0ddf19141ead_.py deleted file mode 100644 index a14492e..0000000 --- a/migrations/versions/0ddf19141ead_.py +++ /dev/null @@ -1,37 +0,0 @@ -"""empty message - -Revision ID: 0ddf19141ead -Revises: d2033352cfdf -Create Date: 2017-03-04 13:42:49.178781 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = '0ddf19141ead' -down_revision = 'd2033352cfdf' -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.create_table('matrix_msg_events', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('event_id', sa.Text(), nullable=True), - sa.Column('room_id', sa.Text(), nullable=True), - sa.Column('pnut_msgid', sa.Text(), nullable=True), - sa.Column('pnut_user', sa.Text(), nullable=True), - sa.Column('pnut_chan', sa.Text(), nullable=True), - sa.Column('deleted', sa.Boolean(), nullable=True), - sa.PrimaryKeyConstraint('id') - ) - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.drop_table('matrix_msg_events') - # ### end Alembic commands ### diff --git a/migrations/versions/744f11d26259_.py b/migrations/versions/744f11d26259_.py deleted file mode 100644 index 0870bd4..0000000 --- a/migrations/versions/744f11d26259_.py +++ /dev/null @@ -1,37 +0,0 @@ -"""empty message - -Revision ID: 744f11d26259 -Revises: f878073e1b4a -Create Date: 2017-05-25 09:51:32.238059 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = '744f11d26259' -down_revision = 'f878073e1b4a' -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.create_table('matrix_room2', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('room_id', sa.Text(), nullable=True), - sa.Column('pnut_chan', sa.Text(), nullable=True), - sa.Column('pnut_since', sa.Text(), nullable=True), - sa.Column('pnut_write', sa.Boolean(), nullable=True), - sa.PrimaryKeyConstraint('id'), - sa.UniqueConstraint('pnut_chan'), - sa.UniqueConstraint('room_id') - ) - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.drop_table('matrix_room2') - # ### end Alembic commands ### diff --git a/migrations/versions/b6667aa4e705_.py b/migrations/versions/b6667aa4e705_.py deleted file mode 100644 index 493eeef..0000000 --- a/migrations/versions/b6667aa4e705_.py +++ /dev/null @@ -1,37 +0,0 @@ -"""empty message - -Revision ID: b6667aa4e705 -Revises: -Create Date: 2017-03-04 11:32:13.728984 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = 'b6667aa4e705' -down_revision = None -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.create_table('matrix_user', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('matrix_id', sa.Text(), nullable=True), - sa.Column('room_id', sa.Text(), nullable=True), - sa.Column('pnut_id', sa.Text(), nullable=True), - sa.Column('pnut_token', sa.Text(), nullable=True), - sa.PrimaryKeyConstraint('id'), - sa.UniqueConstraint('matrix_id'), - sa.UniqueConstraint('pnut_id') - ) - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.drop_table('matrix_user') - # ### end Alembic commands ### diff --git a/migrations/versions/d2033352cfdf_.py b/migrations/versions/d2033352cfdf_.py deleted file mode 100644 index bf60e8d..0000000 --- a/migrations/versions/d2033352cfdf_.py +++ /dev/null @@ -1,36 +0,0 @@ -"""empty message - -Revision ID: d2033352cfdf -Revises: b6667aa4e705 -Create Date: 2017-03-04 12:52:51.625451 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = 'd2033352cfdf' -down_revision = 'b6667aa4e705' -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.create_table('matrix_room', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('room_id', sa.Text(), nullable=True), - sa.Column('pnut_chan', sa.Text(), nullable=True), - sa.Column('pnut_since', sa.Text(), nullable=True), - sa.Column('pnut_write', sa.Boolean(), nullable=True), - sa.PrimaryKeyConstraint('id'), - sa.UniqueConstraint('room_id') - ) - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.drop_table('matrix_room') - # ### end Alembic commands ### diff --git a/migrations/versions/f878073e1b4a_.py b/migrations/versions/f878073e1b4a_.py deleted file mode 100644 index ac683b8..0000000 --- a/migrations/versions/f878073e1b4a_.py +++ /dev/null @@ -1,40 +0,0 @@ -"""empty message - -Revision ID: f878073e1b4a -Revises: 0ddf19141ead -Create Date: 2017-05-03 23:11:07.925047 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = 'f878073e1b4a' -down_revision = '0ddf19141ead' -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.create_table('matrix_admin_rooms', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('matrix_id', sa.Text(), nullable=True), - sa.Column('room_id', sa.Text(), nullable=True), - sa.PrimaryKeyConstraint('id') - ) - op.drop_table('direct_msg_rooms') - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.create_table('direct_msg_rooms', - sa.Column('id', sa.INTEGER(), nullable=False), - sa.Column('sender', sa.TEXT(), nullable=True), - sa.Column('room_id', sa.TEXT(), nullable=True), - sa.PrimaryKeyConstraint('id') - ) - op.drop_table('matrix_admin_rooms') - # ### end Alembic commands ### diff --git a/models.py b/models.py index 75d06df..ecd6fb1 100644 --- a/models.py +++ b/models.py @@ -1,74 +1,32 @@ -from flask_sqlalchemy import SQLAlchemy +from sqlalchemy import Column, ForeignKey, Integer, String, Boolean +from database import Base -db = SQLAlchemy() +class Avatars(Base): + __tablename__ = 'avatars' + id = Column(Integer, primary_key=True) + pnut_user = Column(String(250), unique=True) + avatar = Column(String(250)) -class MatrixUser(db.Model): - id = db.Column(db.Integer, primary_key=True) - matrix_id = db.Column(db.Text, unique=True) - room_id = db.Column(db.Text) - pnut_id = db.Column(db.Text, unique=True) - pnut_token = db.Column(db.Text) +class Rooms(Base): + __tablename__ = 'rooms' + id = Column(Integer, primary_key=True) + room_id = Column(String(250), unique=True) + pnut_chan = Column(Integer, unique=True) + portal = Column(Boolean) - def __init__(self, matrix_id, room_id, pnut_id, pnut_token): - self.matrix_id = matrix_id - self.room_id = room_id - self.pnut_id = pnut_id - self.pnut_token = pnut_token +class Events(Base): + __tablename__ = 'events' + id = Column(Integer, primary_key=True) + event_id = Column(String(250)) + room_id = Column(String(250)) + pnut_msg_id = Column(Integer) + pnut_user_id = Column(Integer) + pnut_chan_id = Column(Integer) + deleted = Column(Boolean) - def __repr__(self): - return '' % self.matrix_id - -class MatrixRoom(db.Model): - id = db.Column(db.Integer, primary_key=True) - room_id = db.Column(db.Text, unique=True) - pnut_chan = db.Column(db.Text) - pnut_since = db.Column(db.Text) - pnut_write = db.Column(db.Boolean, default=True) - - def __init__(self, room_id, pnut_chan, pnut_write=True): - self.room_id = room_id - self.pnut_chan = pnut_chan - self.pnut_write = pnut_write - - def __repr__(self): - return '' % self.room_id - -class MatrixRoom2(db.Model): - id = db.Column(db.Integer, primary_key=True) - room_id = db.Column(db.Text, unique=True) - pnut_chan = db.Column(db.Text, unique=True) - pnut_since = db.Column(db.Text) - pnut_write = db.Column(db.Boolean, default=True) - - def __init__(self, room_id, pnut_chan, pnut_write=True): - self.room_id = room_id - self.pnut_chan = pnut_chan - self.pnut_write = pnut_write - - def __repr__(self): - return '' % self.room_id - -class MatrixMsgEvents(db.Model): - id = db.Column(db.Integer, primary_key=True) - event_id = db.Column(db.Text) - room_id = db.Column(db.Text) - pnut_msgid = db.Column(db.Text) - pnut_user = db.Column(db.Text) - pnut_chan = db.Column(db.Text) - deleted = db.Column(db.Boolean, default=False) - - def __init__(self, event_id, room_id, pnut_msgid, pnut_user, pnut_chan): - self.event_id = event_id - self.room_id = room_id - self.pnut_msgid = pnut_msgid - self.pnut_user = pnut_user - self.pnut_chan = pnut_chan - -class MatrixAdminRooms(db.Model): - id = db.Column(db.Integer, primary_key=True) - matrix_id = db.Column(db.Text) - room_id = db.Column(db.Text) - - def __init__(self, matrix_id, room_id): - self.matrix_id = matrix_id - self.room_id = room_id +class Users(Base): + __tablename__ = 'users' + id = Column(Integer, primary_key=True) + matrix_id = Column(String(250)) + pnut_user_id = Column(Integer) + pnut_user_token = Column(String(250)) diff --git a/pnut-bridge.py b/pnut-bridge.py deleted file mode 100644 index e131f45..0000000 --- a/pnut-bridge.py +++ /dev/null @@ -1,250 +0,0 @@ -import yaml -import logging -import threading -import signal -import time -import datetime -import requests -import json -import pnutpy - -from appservice import app -from models import * - -_shutdown = threading.Event() - -class ChannelMonitor(threading.Thread): - - def __init__(self): - threading.Thread.__init__(self) - pnutpy.api.add_authorization_token(app.config['MATRIX_PNUT_TOKEN']) - self.matrix_api_url = app.config['MATRIX_HOST'] + '/_matrix/client/r0' - self.matrix_api_token = app.config['MATRIX_AS_TOKEN'] - self.txId = 0 - - def generate_matrix_id(self, username): - m_username = app.config['MATRIX_PNUT_PREFIX'] + username - m_userid = "@" + app.config['MATRIX_PNUT_PREFIX'] + username + ":" + app.config['MATRIX_DOMAIN'] - m_display = username + " (pnut)" - return {'username': m_username, 'user_id': m_userid, 'displayname': m_display} - - def is_registered(self, user_id): - url = self.matrix_api_url + '/profile/' + user_id - r = requests.get(url) - if r.status_code == 200: - rdata = r.json() - if 'displayname' in rdata: - return rdata['displayname'] - else: - return False - else: - return False - - def create_matrix_user(self, username): - url = self.matrix_api_url + '/register' - params = { - 'access_token': self.matrix_api_token - } - data = { - 'type': 'm.login.application_service', - 'user': username - } - r = requests.post(url, params=params, data=json.dumps(data)) - if r.status_code == 200: - logging.info('REGISTERED USER: ' + username) - logging.info(r.text) - - def set_displayname(self, muser): - url = self.matrix_api_url + '/profile/' + muser['user_id'] + '/displayname' - params = { - 'access_token': self.matrix_api_token, - 'user_id': muser['user_id'] - } - data = { - 'displayname': muser['displayname'] - } - headers = {'Content-Type': 'application/json'} - r = requests.put(url, params=params, data=json.dumps(data), headers=headers) - - def join_room(self, user_id, roomid): - url = self.matrix_api_url + '/join/' + roomid - params = { - 'access_token': self.matrix_api_token, - 'user_id': user_id - } - r = requests.post(url, params=params) - if r.status_code == 403: - self.invite_room(user_id, roomid) - requests.post(url, params=params) - - def invite_room(self, user_id, roomid): - url = self.matrix_api_url + '/rooms/' + roomid + "/invite" - headers = {"Content-Type": "application/json"} - params = { - 'access_token': self.matrix_api_token, - } - body = { - 'user_id': user_id - } - r = requests.post(url, headers=headers, params=params, data=json.dumps(body)) - - def send_message(self, roomid, msg): - url = self.matrix_api_url + '/rooms/' + roomid +'/send/m.room.message' + '/' + str(self.txId) - #logging.debug('debug: ' + msg['ts']) - #logging.debug('debug: ') - #logging.debug(msg['ts']) - # 'ts': msg['ts'] - params = { - 'access_token': self.matrix_api_token, - 'user_id': msg['user']['user_id'] - } - data = { - 'msgtype': 'm.text', - 'body': msg['text'] - } - headers = {'Content-Type': 'application/json'} - r = requests.put(url, params=params, data=json.dumps(data), headers=headers) - if r.status_code == 200: - eventid = r.json()['event_id'] - el = MatrixMsgEvents(eventid, roomid, msg['msgid'], msg['puser'], msg['chan']) - db.session.add(el) - db.session.commit() - else: - logging.debug('error: ' + str(r.status_code)) - logging.debug(r.text) - - def delete_msgs(self, room, ids): - for i in ids: - r_event = MatrixMsgEvents.query.filter_by(pnut_msgid=i,deleted=False).first() - logging.debug(r_event) - if r_event: - logging.debug(r_event.pnut_msgid) - logging.debug(r_event.event_id) - logging.debug(r_event.deleted) - self.redact_event(room, r_event) - - def redact_event(self, room_id, event): - url = self.matrix_api_url + "/rooms/" + room_id + "/redact/" + event.event_id + "/" + str(self.txId) - headers = {"Content-Type": "application/json"} - params = {'access_token': self.matrix_api_token} - try: - event.deleted = True - db.session.commit() - r = requests.put(url, params=params, data="{}", headers=headers) - logging.debug(r.text) - except Exception as e: - logging.exception('redact_event') - - def update_marker(self, chan, msgid): - try: - response = pnutpy.api.request_json('POST', '/markers', data=[{'name': 'channel:' + chan, 'id': msgid}]) - logging.debug(response) - except Exception as e: - logging.exception('update_marker') - - def poll_channel(self, room): - - try: - messages, meta = pnutpy.api.get_channel_messages(room.pnut_chan, since_id=room.pnut_since, include_raw=1) - # messages, meta = pnutpy.api.get_channel_messages(room.pnut_chan, since_id='last_read', include_raw=1) - logging.debug(meta) - if 'deleted_ids' in meta: - self.delete_msgs(room.room_id, meta['deleted_ids']) - except pnutpy.errors.PnutRateLimitAPIException: - logging.warning('*** Rate limit error while trying to fetch messages! Waiting to retry. ***') - time.sleep(30) - return - except Exception as e: - logging.warning('*** An error occured while trying to fetch messages! Waiting to retry. ***') - logging.exception('poll_channel') - time.sleep(30) - return - - pqueue = [] - for msg in messages: - - # bypass messages posted by the bridge to avoid duplicates and loops - if msg.user.username == app.config['MATRIX_PNUT_USER']: - continue - if msg.source.id == app.config['PNUTCLIENT_ID']: - continue - - if 'content' in msg: - user = self.generate_matrix_id(msg.user.username) - text = msg.content.text + "\n" - # dt = datetime.datetime.strptime(msg.created_at, "%Y-%m-%dT%H:%M:%SZ") - dt = msg.created_at - ts = int(time.mktime(dt.timetuple())) - - for lnk in msg.content.entities.links: - text += "\n" - if 'title' in lnk: - text += lnk.title + "\n" - if 'link' in lnk: - text += lnk.link + "\n" - - for raw in msg.raw: - if raw.type == 'io.pnut.core.oembed': - if 'title' in raw.value: - text += raw.value.title + "\n" - if 'url' in raw.value: - text += raw.value.url + "\n" - - # queue the message in reverse order (because of how pnut returns the messages) - pqueue.insert(0, {'user':user,'text':text,'ts':ts,'msgid':msg.id,'puser':msg.user.username,'chan':room.pnut_chan}) - - # update the last message id - if len(messages) > 0: - room.pnut_since = messages[0].id - db.session.commit() - - # update the stream marker - if 'max_id' in meta: - self.update_marker(room.pnut_chan, meta.max_id) - - # empty the queue to the matrix room - for item in pqueue: - self.txId += 1 - - d = self.is_registered(item['user']['user_id']) - if not d: - self.create_matrix_user(item['user']['username']) - - if d != item['user']['displayname']: - self.set_displayname(item['user']) - - self.join_room(item['user']['user_id'], room.room_id) - time.sleep(1) - - self.send_message(room.room_id, item) - - - def run(self): - logging.info("-- Starting channel monitor --") - app.app_context().push() - rooms = MatrixRoom2.query.all() - if rooms: - self.txId = int(rooms[0].pnut_since) - while not _shutdown.isSet(): - rooms = MatrixRoom2.query.all() - for r in rooms: - self.poll_channel(r) - time.sleep(.5) - time.sleep(3) - logging.info("-- Stopping channel monitor --") - -if __name__ == '__main__': - - with open("config.yaml", "rb") as config_file: - config = yaml.load(config_file) - - logging.basicConfig(level=logging.INFO) - - app.config.update(config) - - monitor = ChannelMonitor() - monitor.start() - app.run(port=config['LISTEN_PORT']) - - _shutdown.set() - logging.info("-- Shutdown in progress --") diff --git a/pnut-matrix-bot.py b/pnut-matrix-bot.py index a05655d..9d58f52 100644 --- a/pnut-matrix-bot.py +++ b/pnut-matrix-bot.py @@ -6,9 +6,9 @@ import pnutpy from matrix_bot_api.matrix_bot_api import MatrixBotAPI from matrix_bot_api.mregex_handler import MRegexHandler from matrix_bot_api.mcommand_handler import MCommandHandler - -from appservice import app -from models import * +from models import Avatars, Rooms, Events, Users +from database import db_session +from sqlalchemy import and_ def help_cb(room, event): @@ -42,14 +42,13 @@ def save_cb(room, event): try: response, meta = pnutpy.api.get_user('me') - with app.app_context(): - user = MatrixUser( - matrix_id=event['sender'], - room_id='', - pnut_id=response['username'], - pnut_token=args[1]) - db.session.add(user) - db.session.commit() + user = Users( + matrix_id=event['sender'], + pnut_user_id=response.id, + pnut_user_token=args[1] + ) + db_session.add(user) + db_session.commit() reply = "Success! You are now authorized as " + response['username'] @@ -64,12 +63,13 @@ def save_cb(room, event): def drop_cb(room, event): try: - with app.app_context(): - user = MatrixUser.query.filter_by(matrix_id=event['sender']).first() - db.session.delete(user) - db.session.commit() - - reply = "Success! Your auth token has been removed." + user = Users.query.filter(Users.matrix_id=event['sender']).one_or_none() + if user is not None: + db_session.delete(user) + db_session.commit() + reply = "Success! Your auth token has been removed." + else: + reply = "You do not appear to be registered." except Exception as e: logging.exception('!drop') @@ -79,17 +79,13 @@ def drop_cb(room, event): def status_cb(room, event): try: - with app.app_context(): - user = MatrixUser.query.filter_by(matrix_id=event['sender']).first() - logging.debug('-- got something --') - logging.debug(user) - + user = Users.query.filter(Users.matrix_id=event['sender']).one_or_none() if user is None: reply = "You are currently not authorized on pnut.io" else: - pnutpy.api.add_authorization_token(user.pnut_token) + pnutpy.api.add_authorization_token(user.pnut_user_token) response, meta = pnutpy.api.get_user('me') - reply = "You are currently authorized as " + response['username'] + reply = "You are currently authorized as " + response.username except pnutpy.errors.PnutAuthAPIException as e: reply = "You are currently not authorized on pnut.io" @@ -108,8 +104,6 @@ if __name__ == "__main__": with open("config.yaml", "rb") as config_file: config = yaml.load(config_file) - app.config.update(config) - bot = MatrixBotAPI(config['TBOT_USER'], config['TBOT_PASS'], config['MATRIX_HOST']) bot.add_handler(MCommandHandler("help", help_cb)) diff --git a/pnut-matrix.py b/pnut-matrix.py new file mode 100644 index 0000000..5ff4464 --- /dev/null +++ b/pnut-matrix.py @@ -0,0 +1,313 @@ +import websocket +import threading +import time +import logging +import yaml +import json +import pnutpy +import requests +import magic + +from matrix_client.api import MatrixHttpApi +from matrix_client.api import MatrixError, MatrixRequestError +from models import Avatars, Rooms, Events +from database import db_session, init_db +from sqlalchemy import and_ +from appservice import app + +logger = logging.getLogger() + +_shutdown = threading.Event() + +def new_message(msg): + logger.debug("channel: " + msg.channel_id) + logger.debug("username: " + msg.user.username) + if 'name' in msg.user: + logger.debug("name: " + msg.user.name) + logger.debug("text: " + msg.content.text) + + # ignore messages posted by the bridge + if msg.user.username == config['MATRIX_PNUT_USER']: + return + + if msg.source.id == config['PNUTCLIENT_ID']: + return + + room = Rooms.query.filter(Rooms.pnut_chan == msg.channel_id).one_or_none() + logger.debug(room) + if room is None: + logger.debug('-not_mapped-') + return + + matrix_id = matrix_id_from_pnut(msg.user.username) + matrix_api = MatrixHttpApi(config['MATRIX_HOST'], + token=config['MATRIX_AS_TOKEN'], + identity=matrix_id) + + profile = get_matrix_profile(matrix_id) + if not profile: + new_matrix_user(msg.user.username) + logger.debug('-new_user-') + profile = {'displayname': None} + + if profile['displayname'] != matrix_display_from_pnut(msg.user): + set_matrix_display(msg.user) + logger.debug('-set_display-') + + avatar = Avatars.query.filter(Avatars.pnut_user == msg.user.username).one_or_none() + if avatar is None or avatar.avatar != msg.user.content.avatar_image.link: + set_matrix_avatar(msg.user) + logger.debug('-set_avatar-') + + # members = matrix_api.get_room_members(room.room_id) + # logger.debug(members) + # join_room(room.room_id, config['MATRIX_AS_ID']) + # TODO: sort out room invite and join logic + join_room(room.room_id, matrix_id) + + if 'content' in msg: + text = msg.content.text + "\n" + ts = int(msg.created_at.strftime('%s')) * 1000 + + lnktext = "" + for link in msg.content.entities.links: + if 'title' in link: + lnktext += link.title + "\n" + if 'link' in link: + lnktext += link.link + "\n" + + if len(lnktext) > 0: + text += "\n" + lnktext + + r = matrix_api.send_message(room.room_id, text, timestamp=ts) + event = Events( + event_id=r['event_id'], + room_id=room.room_id, + pnut_msg_id=msg.id, + pnut_user_id=msg.user.id, + pnut_chan_id=msg.channel_id, + deleted=False) + db_session.add(event) + db_session.commit() + + if len(msg.raw) > 0: + logger.debug('-handle media uploads-') + new_media(room.room_id, msg) + +def new_media(room_id, msg): + matrix_id = matrix_id_from_pnut(msg.user.username) + matrix_api = MatrixHttpApi(config['MATRIX_HOST'], + token=config['MATRIX_AS_TOKEN'], + identity=matrix_id) + ts = int(msg.created_at.strftime('%s')) * 1000 + + for item in msg.raw: + if item.type == 'io.pnut.core.oembed' and 'url' in item.value: + + dl = requests.get(item.value.url, stream=True) + dl.raise_for_status() + with magic.Magic(flags=magic.MAGIC_MIME_TYPE) as m: + mtype = m.id_buffer(dl.content) + info = {'mimetype': mtype} + + ul = matrix_api.media_upload(dl.content, mtype) + + if item.value.type == 'photo': + msgtype = 'm.image' + info['h'] = item.value.height + info['w'] = item.value.width + info['size'] = len(dl.content) + elif item.value.type == 'video' or item.value.type == 'html5video': + msgtype = 'm.video' + info['h'] = item.value.height + info['w'] = item.value.width + info['size'] = len(dl.content) + elif item.value.type == 'audio': + msgtype = 'm.audio' + info['duration'] = int(item.value.duration) * 1000 + info['size'] = len(dl.content) + else: + msgtype = 'm.file' + info['size'] = len(dl.content) + + r = matrix_api.send_content(room_id, ul['content_uri'], item.value.title, msgtype, extra_information=info, timestamp=ts) + event = Events( + event_id=r['event_id'], + room_id=room_id, + pnut_msg_id=msg.id, + pnut_user_id=msg.user.id, + pnut_chan_id=msg.channel_id, + deleted=False) + db_session.add(event) + db_session.commit() + +def delete_message(msg): + matrix_id = matrix_id_from_pnut(msg.user.username) + matrix_api = MatrixHttpApi(config['MATRIX_HOST'], + token=config['MATRIX_AS_TOKEN'], + identity=matrix_id) + + events = Events.query.filter(and_(Events.pnut_msg_id == msg.id, Events.deleted == False)).all() + for event in events: + matrix_api.redact_event(event.room_id, event.event_id) + event.deleted = True + db_session.commit() + +def matrix_id_from_pnut(username): + return "@" + config['MATRIX_PNUT_PREFIX'] + username + ":" + config['MATRIX_DOMAIN'] + +def matrix_display_from_pnut(user): + if 'name' in user: + display = user.name + " <@" + user.username + "> (pnut)" + else: + display = "@" + user.username + return display + # return user.username + " (pnut)" + +def get_matrix_profile(matrix_id): + url = matrix_url + '/profile/' + matrix_id + r = requests.get(url) + + if r.status_code == 200: + return r.json() + else: + return None + +def set_matrix_display(user): + matrix_id = matrix_id_from_pnut(user.username) + matrix_api = MatrixHttpApi(config['MATRIX_HOST'], + token=config['MATRIX_AS_TOKEN'], + identity=matrix_id) + matrix_api.set_display_name(matrix_id, matrix_display_from_pnut(user)) + +def set_matrix_avatar(user): + matrix_id = matrix_id_from_pnut(user.username) + matrix_api = MatrixHttpApi(config['MATRIX_HOST'], + token=config['MATRIX_AS_TOKEN'], + identity=matrix_id) + + dl = requests.get(user.content.avatar_image.link, stream=True) + dl.raise_for_status() + with magic.Magic(flags=magic.MAGIC_MIME_TYPE) as m: + mtype = m.id_buffer(dl.content) + ul = matrix_api.media_upload(dl.content, mtype) + + try: + matrix_api.set_avatar_url(matrix_id, ul['content_uri']) + avatar = Avatars(pnut_user=user.username, avatar=user.content.avatar_image.link) + db_session.add(avatar) + db_session.commit() + + except MatrixRequestError: + logger.exception('failed to set user avatar') + +def new_matrix_user(username): + matrix_api = MatrixHttpApi(config['MATRIX_HOST'], + token=config['MATRIX_AS_TOKEN']) + data = { + 'type': 'm.login.application_service', + 'user': config['MATRIX_PNUT_PREFIX'] + username + } + matrix_api.register(content=data) + +def join_room(room_id, matrix_id): + matrix_api_as = MatrixHttpApi(config['MATRIX_HOST'], + token=config['MATRIX_AS_TOKEN']) + matrix_api = MatrixHttpApi(config['MATRIX_HOST'], + token=config['MATRIX_AS_TOKEN'], + identity=matrix_id) + + try: + matrix_api.join_room(room_id) + + except MatrixRequestError as e: + if e.code == 403: + matrix_api_as.invite_user(room_id, matrix_id) + matrix_api.join_room(room_id) + else: + logger.exception('failed to join room') + + logger.debug('-room_join-') + +def on_message(ws, message): + # logger.debug("on_message: " + message) + msg = json.loads(message) + logger.debug(msg['meta']) + + if 'data' in msg: + + if msg['meta']['type'] == "message": + + # TODO: bypassed other channel types for now + if msg['meta']['channel_type'] != 'io.pnut.core.chat': + return + + pmsg = pnutpy.models.Message.from_response_data(msg['data']) + + if 'is_deleted' in msg['meta']: + if msg['meta']['is_deleted']: + logger.debug("message: delete") + delete_message(pmsg) + else: + logger.debug("uh whut?") + else: + new_message(pmsg) + +def on_error(ws, error): + logger.debug("on_error: !!! ERROR !!!") + logger.error(error) + +def on_close(ws): + logger.debug("on_close: ### CLOSED ###") + if not _shutdown.set(): + time.sleep(10) + t.start() + else: + time.sleep(2) + +def on_open(ws): + + def run(*args): + while not _shutdown.isSet(): + time.sleep(3) + ws.send(".") + time.sleep(1) + ws.close() + logger.debug("*** terminate ***") + + t = threading.Thread(target=run) + t.start() + +if __name__ == '__main__': + # websocket.enableTrace(True) + logging.basicConfig(level=logging.INFO) + + with open("config.yaml", "rb") as config_file: + config = yaml.load(config_file) + + ws_url = 'wss://stream.pnut.io/v0/app?access_token=' + ws_url += config['PNUT_APPTOKEN'] + '&key=' + config['PNUT_APPKEY'] + ws_url += '&include_raw=1' + matrix_url = config['MATRIX_HOST'] + '/_matrix/client/r0' + + # setup the database connection + init_db() + + # setup the websocket connection + ws = websocket.WebSocketApp(ws_url, on_message=on_message, + on_error=on_error, on_close=on_close) + ws.on_open = on_open + t = threading.Thread(target=ws.run_forever) + t.daemon = True + t.start() + + # setup the matrix app service + if config['MATRIX_ADMIN_ROOM']: + logger.debug("- sould join admin room -") + join_room(config['MATRIX_ADMIN_ROOM'], config['MATRIX_AS_ID']) + app.config.update(config) + app.run(port=config['LISTEN_PORT']) + + _shutdown.set() + logger.info('!! shutdown initiated !!') + time.sleep(2) diff --git a/pnutlib.py b/pnutlib.py deleted file mode 100644 index e9f5430..0000000 --- a/pnutlib.py +++ /dev/null @@ -1,273 +0,0 @@ -import requests -import json - -# API root: https://api.pnut.io/v0 - -class Pnut: - - def __init__(self, token=""): - self.token = token - self.username = self._getme() - - def logout(self): - url = "https://api.pnut.io/v0/token"; - headers = { "Authorization": "Bearer " + self.token } - r = requests.delete(url, headers=headers) - return r - - def get_stream(self,since_id=None): - url = "https://api.pnut.io/v0/posts/streams/me" - headers = { "Authorization": "Bearer " + self.token } - parameters = {} - if since_id: - parameters = { "since_id": since_id } - r = requests.get(url, headers=headers, params=parameters) - return r - - def get_unified_stream(self,since_id=None): - url = "https://api.pnut.io/v0/posts/streams/unified" - headers = { "Authorization": "Bearer " + self.token } - parameters = {} - if since_id: - parameters = { "since_id": since_id } - r = requests.get(url, headers=headers, params=parameters) - return r - - def get_global_stream(self,since_id=None): - url = "https://api.pnut.io/v0/posts/streams/global?include_raw=1" - headers = { "Authorization": "Bearer " + self.token } - parameters = {} - if since_id: - parameters = { "since_id": since_id } - try: - r = requests.get(url, headers=headers, params=parameters) - except requests.exceptions.ConnectionError: - r = None - except: - r = None - raise - return r - - def get_channel_stream(self,chan_id,since_id=None): - url = "https://api.pnut.io/v0/channels/" + str(chan_id) + "/messages?include_raw=1" - headers = { "Authorization": "Bearer " + self.token } - parameters = {} - if since_id: - parameters = { "since_id": since_id } - try: - r = requests.get(url, headers=headers, params=parameters) - except requests.exceptions.ConnectionError: - r = None - except: - r = None - raise - return r - - def get_user(self, uid): - url = "https://api.pnut.io/v0/users/" + str(uid) - headers = { "Authorization": "Bearer " + self.token } - r = requests.get(url, headers=headers) - return r - - def get_post(self, postid): - url = "https://api.pnut.io/v0/posts/" + str(postid) + "?include_raw=1" - headers = { "Authorization": "Bearer " + self.token } - r = requests.get(url, headers=headers) - return r - - def get_bookmarks(self): - url = "https://api.pnut.io/v0/users/me/bookmarks" - headers = { "Authorization": "Bearer " + self.token } - r = requests.get(url, headers=headers) - return r - - def add_bookmark(self, postid): - url = "https://api.pnut.io/v0/posts/" + str(postid) + "/bookmark" - headers = { "Authorization": "Bearer " + self.token } - r = requests.put(url, headers=headers) - return r - - def delete_bookmark(self, postid): - url = "https://api.pnut.io/v0/posts/" + str(postid) + "/bookmark" - headers = { "Authorization": "Bearer " + self.token } - r = requests.delete(url, headers=headers) - return r - - def add_repost(self, postid): - url = "https://api.pnut.io/v0/posts/" + str(postid) + "/repost" - headers = { "Authorization": "Bearer " + self.token } - r = requests.put(url, headers=headers) - return r - - def delete_repost(self, postid): - url = "https://api.pnut.io/v0/posts/" + str(postid) + "/repost" - headers = { "Authorization": "Bearer " + self.token } - r = requests.delete(url, headers=headers) - return r - - def follow_user(self, uid): - url = "https://api.pnut.io/v0/users/" + str(uid) + "/follow" - headers = { "Authorization": "Bearer " + self.token } - r = requests.put(url, headers=headers) - return r - - def unfollow_user(self, uid): - url = "https://api.pnut.io/v0/users/" + str(uid) + "/follow" - headers = { "Authorization": "Bearer " + self.token } - r = requests.delete(url, headers=headers) - return r - - def get_followers(self): - url = "https://api.pnut.io/v0/users/me/followers" - headers = { "Authorization": "Bearer " + self.token } - r = requests.get(url, headers=headers) - return r - - def get_following(self): - url = "https://api.pnut.io/v0/users/me/following" - headers = { "Authorization": "Bearer " + self.token } - r = requests.get(url, headers=headers) - return r - - def get_mentions(self): - url = "https://api.pnut.io/v0/users/me/mentions" - headers = { "Authorization": "Bearer " + self.token } - r = requests.get(url, headers=headers) - return r - - def post(self, text): - url = "https://api.pnut.io/v0/posts" - headers = { "Authorization": "Bearer " + self.token } - body = { - "text": text - } - r = requests.post(url, data=body, headers=headers) - return r - - def channel_post(self, chan_id, text, raw): - url = "https://api.pnut.io/v0/channels/" + str(chan_id) + "/messages" - headers = { - "Authorization": "Bearer " + self.token, - "Content-Type": "application/json" - } - data = {"text": text } - if raw: - url += "?include_raw=1" - data["raw"] = raw - body = json.dumps(data) - r = requests.post(url, data=body, headers=headers) - return r - - def delete_channel_post(self, chan_id, msg_id): - url = "https://api.pnut.io/v0/channels/" + str(chan_id) + "/messages/" + str(msg_id) - headers = { - "Authorization": "Bearer " + self.token, - "Content-Type": "application/json" - } - r = requests.delete(url, data=json.dumps({}), headers=headers) - return r - - def global_post_oembed(self, text, embed): - url = "https://api.pnut.io/v0/posts?include_raw=1" - headers = { - "Authorization": "Bearer " + self.token, - "Content-Type": "application/json" - } - body = json.dumps({"text": text, "raw": embed}) - r = requests.post(url, data=body, headers=headers) - return r - - def process_text(self, text): - url = "https://api.pnut.io/v0/text/process" - headers = { "Authorization": "Bearer " + self.token } - body = { - "text": text - } - r = requests.post(url, json=body, headers=headers) - return r - - def reply(self, postid, text): - url = "https://api.pnut.io/v0/posts" - headers = { "Authorization": "Bearer " + self.token } - me = self.username - author = self._getauthor(postid) - if author == me or author == None: - mtext = text - else: - mtext = "@%s %s" % (author, text) - body = {} - body["text"] = mtext - if len(postid) > 0: - body["reply_to"] = postid - r = requests.post(url, data=body, headers=headers) - return r - - def replyall(self, postid, text): - url = "https://api.pnut.io/v0/posts" - headers = { "Authorization": "Bearer " + self.token } - me = self.username - author = self._getauthor(postid) - mentions = self._getmentions(postid) - mtext = "@%s " % (author) - mentions = [m for m in mentions if m != me] - mentions = [m for m in mentions if m != author] - if mentions: - mtext += " " - for nick in mentions: - mtext += "@%s " % nick - mtext += "" - mtext += text - body = {} - body["text"] = mtext - if len(str(postid)) > 0: - body["reply_to"] = postid - r = requests.post(url, data=body, headers=headers) - return r - - def _getme(self): - if len(self.token) > 0: - url = "https://api.pnut.io/v0/users/me" - headers = { "Authorization": "Bearer " + self.token } - r = requests.get(url, headers=headers) - if r.status_code == 200: - rdata = rdata = json.loads(r.text) - return rdata["data"]["username"] - else: - return None - else: - return None - - def _getpost(self, postid): - url = "https://api.pnut.io/v0/posts/" + str(postid) - headers = { "Authorization": "Bearer " + self.token } - r = requests.get(url, headers=headers) - if r.status_code == 200: - rdata = json.loads(r.text) - else: - return None - return rdata["data"] - - def _getauthor(self, postid): - url = "https://api.pnut.io/v0/posts/" + str(postid) - headers = { "Authorization": "Bearer " + self.token } - r = requests.get(url, headers=headers) - if r.status_code == 200: - rdata = json.loads(r.text) - else: - return None - return rdata["data"]["user"]["username"] - - def _getmentions(self, postid): - url = "https://api.pnut.io/v0/posts/" + str(postid) - headers = { "Authorization": "Bearer " + self.token } - r = requests.get(url, headers=headers) - if r.status_code == 200: - rdata = json.loads(r.text) - else: - return None - mentions = [] - entities = rdata["data"]["content"]["entities"] - if "mentions" in entities: - for m in entities["mentions"]: - mentions.append(m["text"]) - return mentions diff --git a/requirements.txt b/requirements.txt index ee7829d..2a0d308 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,5 +2,8 @@ pyyaml requests matrix-client Flask -Flask-Migrate -Flask-SQLAlchemy \ No newline at end of file +pnutpy +sqlalchemy +websocket-client +filemagic +git+https://github.com/shawnanastasio/python-matrix-bot-api \ No newline at end of file