diff --git a/README.md b/README.md index ab8203b..3501162 100644 --- a/README.md +++ b/README.md @@ -7,19 +7,19 @@ This bridge will pass pnut.io channel messages through to Matrix, and Matrix mes ## Usage -The public bridge is once again online! - -See [Using-the-public-bridge](https://gitlab.com/thrrgilag/pnut-matrix/-/wikis/Using-the-public-bridge) for details. +The public bridge is not yet online pending uplift of this code base. Stay tuned! ## Installation +**Warning! This code is extremely unstable and not yet ready for use on your matrix server.** + Currently pnut-matrix has been only tested with and confirmed to work with [synapse]. Please refer to the [synapse installation instructions] for details on how to setup your homeserver. To install the latest version of pnut-matrix from source: ```sh -git clone https://gitlab.com/thrrgilag/pnut-matrix.git +git clone https://git.dreamfall.space/spacenerdmo/pnut-matrix.git cd pnut-matrix python3 -m venv env source env/bin/activate @@ -43,12 +43,11 @@ curl --data '{"type": "m.login.application_service", "username": "your_sender_lo ## Contributing and support -You can open issues for bugs or feature requests and you can submit merge requests to this project on [GitLab]. You can also submit issues and patches directly to [morgan@mcmillian.dev]. +Please submit bugs, feature requests, and patches to [morgan@mcmillian.dev]. -Join my public chat room for development discussion. +Join my public chat room on pnut.io for development discussion. -- [pnut-matrix on pnut.io] -- [#pnut_999:pnut-matrix.dreamfall.space] +- [pnut-matrix] ## License @@ -58,9 +57,7 @@ GPLv3, see [LICENSE]. [synapse]: https://github.com/matrix-org/synapse [synapse installation instructions]: https://matrix-org.github.io/synapse/latest/setup/installation.html [syanpse configuration]: https://matrix-org.github.io/synapse/latest/application_services.html -[GitLab]: https://gitlab.com/thrrgilag/pnut-matrix/ [morgan@mcmillian.dev]: mailto:morgan@mcmillian.dev -[pnut-matrix on pnut.io]: https://patter.chat/999 -[#pnut_999:pnut-matrix.dreamfall.space]: https://matrix.to/#/#pnut_999:pnut-matrix.dreamfall.space +[pnut-matrix]: https://patter.chat/999 [LICENSE]: LICENSE [^1]: https://github.com/matrix-org/matrix-appservice-irc/issues/1270#issuecomment-849765090 diff --git a/appservice.py b/appservice.py index e9f80d6..2633773 100644 --- a/appservice.py +++ b/appservice.py @@ -1,13 +1,17 @@ import json +import yaml import requests import logging +import logging.config import re import pnutpy import textwrap import time +import os + +from mautrix.client import ClientAPI +from mautrix.types import * -from matrix_client.api import MatrixHttpApi -from matrix_client.api import MatrixError, MatrixRequestError from models import Avatars, Rooms, Events, Users, DirectRooms, ControlRooms from database import db_session from sqlalchemy import and_ @@ -19,11 +23,11 @@ app = Flask(__name__) @app.errorhandler(404) def not_found(error): - return jsonify({'errcode':'COM.MONKEYSTEW.PNUT_NOT_FOUND'}), 404 + return jsonify({'errcode':'PNUT_NOT_FOUND'}), 404 @app.errorhandler(403) def forbidden(error): - return jsonify({'errcode':'COM.MONKEYSTEW.PNUT_FORBIDDEN'}), 403 + return jsonify({'errcode':'PNUT_FORBIDDEN'}), 403 @app.teardown_appcontext def shutdown_session(exception=None): @@ -31,7 +35,8 @@ def shutdown_session(exception=None): @app.route("/_matrix/app/v1/rooms/") @app.route("/rooms/") -def query_alias(alias): +async def query_alias(alias): + logging.debug("--- query alias ---") alias_localpart = alias.split(":")[0][1:] channel_id = int(alias_localpart.split('_')[1]) @@ -42,64 +47,66 @@ def query_alias(alias): token = app.config['MATRIX_PNUT_TOKEN'] pnutpy.api.add_authorization_token(token) try: - channel, meta = pnutpy.api.get_channel(channel_id, include_raw=1) + logging.debug("---- getting the channel ----") + channel, meta = pnutpy.api.get_channel(channel_id, + include_channel_raw=1) if 'is_active' in channel and channel.is_active == False: - logger.debug("-channel isn't active-") + logging.debug("-channel isn't active-") abort(404) - channel_settings = {} - for item in channel.raw: - if item.type == 'io.pnut.core.chat-settings': - channel_settings = item.value + if 'io.pnut.core.chat-settings' in channel.raw: + for setting in channel.raw['io.pnut.core.chat-settings']: + if 'name' in setting: + name = setting['name'] + else: + name = None + if 'description' in setting: + topic = setting['description']['text'] + else: + topic = None + + matrix_api = ClientAPI(app.config['MATRIX_AS_ID'], + base_url=app.config['MATRIX_HOST'], + token=app.config['MATRIX_AS_TOKEN']) - # Matrix sdk doesn't include all details in a single call - room = {'room_alias_name': alias_localpart} - if 'name' in channel_settings: - room['name'] = channel_settings['name'] - if 'description' in channel_settings: - room['topic'] = channel_settings['description'] if channel.acl.read.public: - room['preset'] = 'public_chat' - room['visibility'] = 'public' + visibility = RoomDirectoryVisibility.PUBLIC + preset = RoomCreatePreset.PUBLIC else: - abort(401) + visibility = RoomDirectoryVisibility.PRIVATE + preset = RoomCreatePreset.PRIVATE + room_id = await matrix_api.create_room(alias_localpart, + visibility=visibility, + preset=preset, + name=name, + topic=topic) - url = app.config['MATRIX_HOST'] + '/_matrix/client/api/v1/createRoom' - headers = { - "Content-Type": "application/json", - "Authorization": "Bearer " + app.config['MATRIX_AS_TOKEN'] - } - r = requests.post(url, headers=headers, data=json.dumps(room)) - if r.status_code == 200: + if not channel.you_subscribed: pnutpy.api.subscribe_channel(channel_id) - rdata = r.json() - rr = Rooms( - room_id=rdata['room_id'], - pnut_chan=channel_id, - portal=True - ) - db_session.add(rr) - db_session.commit() - - else: - logger.error("Unable to create room") - logger.error(r.status_code) - logger.error(r.text) - abort(400) + rr = Rooms( + room_id=room_id, + pnut_chan=channel_id, + portal=True + ) + logging.debug(rr.room_id) + logging.debug(rr.pnut_chan) + db_session.add(rr) + db_session.commit() except pnutpy.errors.PnutPermissionDenied: + logging.debug("-permission denied-") abort(401) except Exception: - logger.exception("-couldn't get the pnut channel-") + logging.exception("-couldn't get the pnut channel-") abort(404) return jsonify({}) @app.route("/_matrix/app/v1/transactions/", methods=["PUT"]) @app.route("/transactions/", methods=["PUT"]) -def on_receive_events(transaction): +async def on_receive_events(transaction): access_token = request.args.get('access_token', '') if access_token != app.config['MATRIX_HS_TOKEN']: @@ -107,114 +114,70 @@ def on_receive_events(transaction): events = request.get_json()["events"] for event in events: - logger.debug(event) + logging.debug(event) - if app.config['MATRIX_ADMIN_ROOM'] and app.config['MATRIX_ADMIN_ROOM'] == event['room_id']: - return on_admin_event(event) + if (app.config['MATRIX_ADMIN_ROOM'] and + app.config['MATRIX_ADMIN_ROOM'] == event['room_id']): + await on_admin_event(event) + return jsonify({}) - user = Users.query.filter(Users.matrix_id == event['user_id']).one_or_none() + user = Users.query.filter(Users.matrix_id == + event['sender']).one_or_none() if event['type'] == 'm.room.message': - new_message(event, user) + await new_message(event, user) - elif event['type'] == 'm.sticker': - new_sticker(event, user) + # elif event['type'] == 'm.sticker': + # new_sticker(event, user) elif event['type'] == 'm.room.redaction': delete_message(event, user) elif event['type'] == 'm.room.member': - if 'is_direct' in event['content'] and 'membership' in event['content']: - if event['content']['membership'] == "invite" and event['content']['is_direct']: - return on_direct_invite(event) + if ('is_direct' in event['content'] and + 'membership' in event['content']): + if (event['content']['membership'] == "invite" and + event['content']['is_direct']): + logging.debug('----> direct invite <----') + await on_direct_invite(event) + return jsonify({}) if 'membership' in event['content']: if event['content']['membership'] == "leave": - return on_leave_event(event) + logging.debug('----> leave event <----') + await on_leave_event(event) + return jsonify({}) - logger.debug("----room member event----") - logger.debug(user) - logger.debug(event) + logging.debug("----room member event----") + logging.debug(user) + logging.debug(event) return jsonify({}) -def new_sticker(event, user): - if app.config['MATRIX_PNUT_PREFIX'] in event['user_id'] or 'pnut-bridge' in event['user_id']: - logger.debug('-skipping dup event-') - return - - room = Rooms.query.filter(Rooms.room_id == event['room_id']).one_or_none() - if room is None: - logger.debug('-room not mapped-') - return - - if user is not None: - token = user.pnut_user_token - prefix = "" - else: - token = app.config['MATRIX_PNUT_TOKEN'] - matrix_profile = get_profile(event['user_id']) - if "displayname" in matrix_profile: - prefix = "[" + matrix_profile['displayname'] + "] (" + event['user_id'] + ")\n" - else: - prefix = "(" + event['user_id'] + ")\n" - - pnutpy.api.add_authorization_token(token) - # evtext, evraw = msg_from_event(event) - text = "sticker::" + event['content']['body'] + "\n" - value = {'type': "photo", 'version': "1.0"} - value['url'] = app.config['MATRIX_URL'] + '/_matrix/media/r0/download/' + event['content']['url'][6:] - value['title'] = event['content']['body'] - if 'h' in event['content']['info'] and 'w' in event['content']['info']: - value['height'] = event['content']['info']['h'] - value['width'] = event['content']['info']['h'] - else: - return - raw = {'type': "io.pnut.core.oembed", 'value': value} - embed = [raw] - text = prefix + text - - try: - msg, meta = pnutpy.api.create_message(room.pnut_chan, data={'text': text, 'raw': embed}) - revent = Events( - event_id=event['event_id'], - room_id=event['room_id'], - pnut_msg_id=msg.id, - pnut_user_id=msg.user.id, - pnut_chan_id=room.pnut_chan, - deleted=False - ) - db_session.add(revent) - db_session.commit() - - except pnutpy.errors.PnutAuthAPIException: - logger.exception('-unable to post to pnut channel-') - return - - except Exception: - logger.exception('-something bad happened here-') - return - -def new_message(event, user): - if app.config['MATRIX_PNUT_PREFIX'] in event['user_id'] or 'pnut-bridge' in event['user_id']: - logger.debug('-skipping dup event-') +async def new_message(event, user): + if (app.config['MATRIX_PNUT_PREFIX'] in event['sender'] or + 'pnut-bridge' in event['sender']): + logging.debug('-skipping dup event-') return if 'msgtype' not in event['content']: - logger.debug('-unknown message type-') + logging.debug('-unknown message type-') return - control = ControlRooms.query.filter(ControlRooms.room_id == event['room_id']).one_or_none() + control = ControlRooms.query.filter(ControlRooms.room_id == + event['room_id']).one_or_none() if control is not None: - return on_control_message(event) + await on_control_message(event) + return - direct = DirectRooms.query.filter(DirectRooms.room_id == event['room_id']).one_or_none() + direct = DirectRooms.query.filter(DirectRooms.room_id == + event['room_id']).one_or_none() if direct is not None: return on_direct_message(event, user, direct) room = Rooms.query.filter(Rooms.room_id == event['room_id']).one_or_none() if room is None: - logger.debug('-room not mapped-') + logging.debug('-room not mapped-') return if user is not None: @@ -222,20 +185,25 @@ def new_message(event, user): prefix = "" else: token = app.config['MATRIX_PNUT_TOKEN'] - matrix_profile = get_profile(event['user_id']) + matrix_profile = get_profile(event['sender']) if "displayname" in matrix_profile: - prefix = "[" + matrix_profile['displayname'] + "] (" + event['user_id'] + ")\n" + prefix = (f"[{matrix_profile['displayname']}]" + f" ({event['sender']})\n") else: - prefix = "(" + event['user_id'] + ")\n" + prefix = "(" + event['sender'] + ")\n" pnutpy.api.add_authorization_token(token) - embed = [crosspost_raw(event)] - evtext, evraw = msg_from_event(event) - text = prefix + evtext - if len(evraw) > 0: - embed.append(evraw) + raw = {} + raw['io.pnut.core.crosspost'] = [crosspost_raw(event)] + text, oembed = msg_from_event(event) + text = prefix + text + if oembed: + raw['io.pnut.core.oembed'] = [oembed] + logging.debug(oembed) + try: - msg, meta = pnutpy.api.create_message(room.pnut_chan, data={'text': text, 'raw': embed}) + msg, meta = pnutpy.api.create_message(room.pnut_chan, + data={'text': text, 'raw': raw}) revent = Events( event_id=event['event_id'], room_id=event['room_id'], @@ -247,136 +215,142 @@ def new_message(event, user): db_session.add(revent) db_session.commit() - if user is not None: - cctag = re.search('##$', text) - if cctag: - raw = [] - cname = get_channel_settings(room.pnut_chan)['name'] - text = text[:-2] - ftext = '\n\n[' + cname + "](https://patter.chat/room.html?channel=" + str(room.pnut_chan) + ")" - mtext = textwrap.wrap(text + ftext, 254) - if len(mtext) > 1: - longpost = { - 'title': "", - 'body': text, - 'tstamp': time.time() * 1000 - } - pretext = textwrap.wrap(text, 100) - text = pretext[0] - text += "... - https://longpo.st/p/{object_id} - #longpost" - raw.append({'type':"nl.chimpnut.blog.post", 'value': longpost}) - - text += ftext - r, meta = pnutpy.api.create_post(data={'text': text, 'raw': raw}) + # TODO: need to redo this for global message + # if user is not None: + # cctag = re.search('##$', text) + # if cctag: + # raw = [] + # cname = get_channel_settings(room.pnut_chan)['name'] + # text = text[:-2] + # ftext = '\n\n[' + cname + "](https://patter.chat/room.html?channel=" + str(room.pnut_chan) + ")" + # mtext = textwrap.wrap(text + ftext, 254) + # if len(mtext) > 1: + # longpost = { + # 'title': "", + # 'body': text, + # 'tstamp': time.time() * 1000 + # } + # pretext = textwrap.wrap(text, 100) + # text = pretext[0] + # text += "... - https://longpo.st/p/{object_id} - #longpost" + # raw.append({'type':"nl.chimpnut.blog.post", 'value': longpost}) + # + # text += ftext + # r, meta = pnutpy.api.create_post(data={'text': text, 'raw': raw}) except pnutpy.errors.PnutAuthAPIException: - logger.exception('-unable to post to pnut channel-') + logging.exception('-unable to post to pnut channel-') return except Exception: - logger.exception('-something bad happened here-') + logging.exception('-something bad happened here-') return def msg_from_event(event): text = None - raw = {} - if event['content']['msgtype'] == 'm.text' or event['content']['msgtype'] == 'm.notice': + oembed = None + if (event['content']['msgtype'] == 'm.text' or + event['content']['msgtype'] == 'm.notice'): text = event['content']['body'] elif event['content']['msgtype'] == 'm.emote': text = "* " + event['content']['body'] - elif event['content']['msgtype'] == 'm.image': - text = event['content']['body'] + "\n" - text += app.config['MATRIX_URL'] + '/_matrix/media/r0/download/' + event['content']['url'][6:] - raw = raw_from_event(event) - - elif event['content']['msgtype'] == 'm.video': - text = event['content']['body'] + "\n" - text += app.config['MATRIX_URL'] + '/_matrix/media/r0/download/' + event['content']['url'][6:] - - elif event['content']['msgtype'] == 'm.audio': - text = event['content']['body'] + "\n" - text += app.config['MATRIX_URL'] + '/_matrix/media/r0/download/' + event['content']['url'][6:] + elif (event['content']['msgtype'] == 'm.image' or + event['content']['msgtype'] == 'm.video' or + event['content']['msgtype'] == 'm.audio'): + oembed = oembed_from_event(event) + if ('title' in oembed and 'url' in oembed): + text = (f"[{oembed['title']}]" + f"({oembed['url']})") elif event['content']['msgtype'] == 'm.file': - text = event['content']['body'] + "\n" - text += app.config['MATRIX_URL'] + '/_matrix/media/r0/download/' + event['content']['url'][6:] + file_url = event['content']['url'][6:] + file_name = event['content']['body'] + dl_url = (f"{app.config['MATRIX_URL']}" + f"/_matrix/client/v1/media/download/{file_url}" + f"/{file_name}") + text = (f"[{file_name}]" + f"({dl_url})") else: - logger.debug('-unknown msg type- ' + event['content']['msgtype']) + logging.debug('-unknown msg type- ' + event['content']['msgtype']) return - return text, raw + return text, oembed def crosspost_raw(event): - cross_profile = {'username': event['user_id']} - matrix_profile = get_profile(event['user_id']) + cross_profile = {'username': event['sender']} + matrix_profile = get_profile(event['sender']) if "avatar_url" in matrix_profile: - cross_profile['avatar_image'] = app.config['MATRIX_URL'] + '/_matrix/media/r0/download/' + matrix_profile['avatar_url'][6:] + cross_profile['avatar_image'] = (f"{app.config['MATRIX_URL']}" + f"/_matrix/media/r0/download/" + f"{matrix_profile['avatar_url'][6:]}") - embed = { - 'type': "io.pnut.core.crosspost", - 'value': { - 'canonical_url': "https://matrix.to/#/" + event['room_id'] + "/" + event['event_id'] + ":" + app.config['MATRIX_DOMAIN'], - 'source': { - 'name': "matrix", - 'url': "https://matrix.org" - }, - 'user': cross_profile - } - } - return embed + crosspost = {} + crosspost['canonical_url'] = (f"https://matrix.to/#/{event['room_id']}" + f"/{event['event_id']}" + f":{app.config['MATRIX_DOMAIN']}") + crosspost['source'] = {'name': "matrix.", 'url': "https://matrix.org"} + crosspost['user'] = cross_profile -def raw_from_event(event): - url = app.config['MATRIX_URL'] + '/_matrix/media/r0/download/' + event['content']['url'][6:] + return crosspost +# TODO: This could be used for uploading the media to pnut, maybe +# async def media_from_event(event): +# matrix_api = ClientAPI(app.config['MATRIX_AS_ID'], +# base_url=app.config['MATRIX_HOST'], +# token=app.config['MATRIX_AS_TOKEN']) +# +# mxc_url = event['content']['url'] +# media_file = await matrix_api.download_media(mxc_url) + +def oembed_from_event(event): + media_url = event['content']['url'][6:] + file_name = event['content']['body'] + dl_url = (f"{app.config['MATRIX_URL']}" + f"/_matrix/client/v1/media/download/{media_url}" + f"/{file_name}") + + oembed = {} if event['content']['msgtype'] == 'm.image': - value = {'type': "photo", 'version': "1.0"} - value['url'] = url - value['title'] = event['content']['body'] + oembed['provider_name'] = "matrix" + oembed['provider_url'] = "https://matrix.org" + oembed['version'] = "1.0" + oembed['type'] = "photo" + oembed['title'] = file_name + oembed['url'] = dl_url if 'info' in event['content']: - if 'w' in event['content']['info']: - value['width'] = event['content']['info']['w'] - else: - value['width'] = 200 if 'h' in event['content']['info']: - value['height'] = event['content']['info']['h'] - else: - value['height'] = 200 - if 'thumbnail_info' in event['content']['info']: - if 'thumbnail_url' in event['content']['info']: - value['thumbnail_url'] = app.config['MATRIX_URL'] + '/_matrix/media/r0/download/' + event['content']['info']['thumbnail_url'][6:] - if 'w' in event['content']['info']['thumbnail_info']: - value['thumbnail_width'] = event['content']['info']['thumbnail_info']['w'] - else: - value['thumbnail_width'] = 200 - if 'h' in event['content']['info']['thumbnail_info']: - value['thumbnail_height'] = event['content']['info']['thumbnail_info']['h'] - else: - value['thumbnail_height'] = 200 + oembed['height'] = event['content']['info']['h'] + if 'w' in event['content']['info']: + oembed['width'] = event['content']['info']['w'] elif event['content']['msgtype'] == 'm.video': - # TODO: Need to sort out the oembed for this media type - value = {'type': "html5video", 'version': "1.0"} - source = {'url': url} - value['title'] = event['content']['body'] + oembed['provider_name'] = "matrix" + oembed['provider_url'] = "https://matrix.org" + oembed['version'] = "1.0" + oembed['type'] = "video" + oembed['title'] = file_name + oembed['url'] = dl_url if 'info' in event['content']: - value['width'] = event['content']['info']['w'] - value['height'] = event['content']['info']['h'] - source['type'] = event['content']['info']['mimetype'] - else: - return None - value['sources'] = [source] + if 'h' in event['content']['info']: + oembed['height'] = event['content']['info']['h'] + if 'w' in event['content']['info']: + oembed['width'] = event['content']['info']['w'] elif event['content']['msgtype'] == 'm.audio': - # TODO: Need to sort out the oembed for this media type - value = {'type': "audio", 'version': "1.0"} - return None - else: - return None + oembed['provider_name'] = "matrix" + oembed['provider_url'] = "https://matrix.org" + oembed['version'] = "1.0" + oembed['type'] = "audio" + oembed['title'] = file_name + oembed['url'] = dl_url + if 'info' in event['content']: + if 'duration' in event['content']['info']: + oembed['duration'] = event['content']['info']['duration'] - return {'type': "io.pnut.core.oembed", 'value': value} + return oembed def delete_message(event, user): @@ -388,9 +362,10 @@ def delete_message(event, user): token = app.config['MATRIX_PNUT_TOKEN'] pnutpy.api.add_authorization_token(token) - e = Events.query.filter(and_(Events.event_id == event['redacts'], Events.deleted == False)).one_or_none() + e = Events.query.filter(and_(Events.event_id == event['redacts'], + Events.deleted == False)).one_or_none() if e is None: - logger.debug("- can't find the event to remove -") + logging.debug("- can't find the event to remove -") return try: @@ -418,73 +393,74 @@ def get_channel_settings(channel_id): channel_settings = item.value except Exception: - logger.exception('-unable to get channel settings-') + logging.exception('-unable to get channel settings-') return channel_settings -def create_room(channel, user): - channel_settings = {} - for item in channel.raw: - if item.type == 'io.pnut.core.chat-settings': - channel_settings = item.value +async def create_pnut_matrix_room(channel, user): + name = None + topic = None + alias_localpart = f"{app.config['MATRIX_PNUT_PREFIX']}{channel.id}" + invitees = [user.matrix_id] - # Matrix sdk doesn't include all details in a single call - room = {'room_alias_name': app.config['MATRIX_PNUT_PREFIX'] + channel.id} - logger.debug(user) - logger.debug(room) - room['invite'] = [user.matrix_id] - if 'name' in channel_settings: - room['name'] = channel_settings['name'] - if 'description' in channel_settings: - room['topic'] = channel_settings['description'] if channel.acl.read.public: - room['preset'] = 'public_chat' - room['visibility'] = 'public' - elif channel.acl.read.any_user or channel.acl.read.you: - room['preset'] = 'private_chat' - room['visibility'] = 'private' + visibility = RoomDirectoryVisibility.PUBLIC + preset = RoomCreatePreset.PUBLIC else: - abort(401) + visibility = RoomDirectoryVisibility.PRIVATE + preset = RoomCreatePreset.PRIVATE - url = app.config['MATRIX_HOST'] + '/_matrix/client/api/v1/createRoom' + if 'io.pnut.core.chat-settings' in channel.raw: + for setting in channel.raw['io.pnut.core.chat-settings']: + if 'name' in setting: + name = setting['name'] + if 'description' in setting: + topic = setting['description']['text'] + + matrix_api = ClientAPI(app.config['MATRIX_AS_ID'], + base_url=app.config['MATRIX_HOST'], + token=app.config['MATRIX_AS_TOKEN']) + + room_id = await matrix_api.create_room(alias_localpart, + invitees=invitees, + visibility=visibility, + preset=preset, + name=name, + topic=topic) + + rr = Rooms(room_id=room_id, pnut_chan=channel.id, portal=True) + db_session.add(rr) + db_session.commit() + +def new_matrix_user(username): + endpoint = "/_matrix/client/v3/register" + url = app.config['MATRIX_HOST'] + endpoint + params = {'kind': 'user'} + data = { + 'type': 'm.login.application_service', + 'username': app.config['MATRIX_PNUT_PREFIX'] + username + } headers = { "Content-Type": "application/json", "Authorization": "Bearer " + app.config['MATRIX_AS_TOKEN'] } - r = requests.post(url, headers=headers, data=json.dumps(room)) - + r = requests.post(url, headers=headers, json=data, params=params) if r.status_code == 200: - #pnutpy.api.subscribe_channel(channel.id) - rdata = r.json() - rr = Rooms( - room_id=rdata['room_id'], - pnut_chan=channel.id, - portal=True - ) - db_session.add(rr) - db_session.commit() + return else: - logger.error("Unable to create room") - logger.error(r.status_code) - logger.error(r.text) - abort(400) + errmsg = f"- unable to register {username} -" + logging.warning(errmsg) + logging.debug(r.status_code) + logging.debug(r.text) + return -def new_matrix_user(username): - matrix_api = MatrixHttpApi(app.config['MATRIX_HOST'], - token=app.config['MATRIX_AS_TOKEN']) - data = {'type': 'm.login.application_service','username': app.config['MATRIX_PNUT_PREFIX'] + username} - try: - matrix_api.register(content=data) +async def on_admin_event(event): + matrix_api = ClientAPI(app.config['MATRIX_AS_ID'], + base_url=app.config['MATRIX_HOST'], + token=app.config['MATRIX_AS_TOKEN']) - except Exception: - errmsg = "- new_matrix_user user already exists -" - logger.warning(errmsg) - -def on_admin_event(event): - matrix_api = MatrixHttpApi(app.config['MATRIX_HOST'], - token=app.config['MATRIX_AS_TOKEN']) - logger.debug("- admin room event recieved -") + logging.debug("- admin room event recieved -") if event['type'] != 'm.room.message': return jsonify({}) @@ -494,33 +470,36 @@ def on_admin_event(event): try: if msg[0] == 'help': if len(msg) > 1: - matrix_api.send_message(event['room_id'], cmd_admin_help(msg[1])) + await matrix_api.send_message(event['room_id'], + cmd_admin_help(msg[1])) else: - matrix_api.send_message(event['room_id'], cmd_admin_help()) + await matrix_api.send_message(event['room_id'], + cmd_admin_help()) elif msg[0] == 'list': - matrix_api.send_message(event['room_id'], cmd_admin_list()) + await matrix_api.send_message(event['room_id'], cmd_admin_list()) elif msg[0] == 'unlink': if len(msg) > 1: - matrix_api.send_message(event['room_id'], cmd_admin_unlink(msg[1])) + await matrix_api.send_message(event['room_id'], + cmd_admin_unlink(msg[1])) else: - matrix_api.send_message(event['room_id'], cmd_admin_help('unlink')) + await matrix_api.send_message(event['room_id'], + cmd_admin_help('unlink')) elif msg[0] == 'link': if len(msg) > 2: - matrix_api.send_message(event['room_id'], cmd_admin_link(msg[1], msg[2])) + await matrix_api.send_message(event['room_id'], + cmd_admin_link(msg[1], msg[2])) else: - matrix_api.send_message(event['room_id'], cmd_admin_help('link')) + await matrix_api.send_message(event['room_id'], + cmd_admin_help('link')) except Exception: errmsg = "- on_admin_event -" - logger.exception(errmsg) - - return jsonify({}) + logging.exception(errmsg) def cmd_admin_help(cmd=None): - help_usage = "help [command]" help_desc = "Show information about available commands." list_usage = "list" @@ -549,7 +528,7 @@ def cmd_admin_help(cmd=None): text += unlink_usage + "\n" text += link_usage + "\n" - return text + return TextMessageEventContent(msgtype='m.text', body=text) def cmd_admin_list(): text = "" @@ -568,22 +547,24 @@ def cmd_admin_list(): text += "(portal)" text += '\n' - return text + return TextMessageEventContent(msgtype='m.text', body=text) -def cmd_admin_link(room_id, pnut_chan_id): - matrix_api = MatrixHttpApi(app.config['MATRIX_HOST'], - token=app.config['MATRIX_AS_TOKEN']) +async def cmd_admin_link(room_id, pnut_chan_id): + matrix_api = ClientAPI(app.config['MATRIX_AS_ID'], + base_url=app.config['MATRIX_HOST'], + token=app.config['MATRIX_AS_TOKEN']) pnutpy.api.add_authorization_token(app.config['MATRIX_PNUT_TOKEN']) mrcheck = Rooms.query.filter(Rooms.room_id == room_id).one_or_none() pncheck = Rooms.query.filter(Rooms.pnut_chan == pnut_chan_id).one_or_none() if mrcheck is not None or pncheck is not None: - return "- room may already be linked -" + text = "- room may already be linked -" + return TextMessageEventContent(msgtype='m.text', body=text) try: channel, meta = pnutpy.api.subscribe_channel(pnut_chan_id) - r = matrix_api.join_room(room_id) + await matrix_api.join_room(room_id) rec = Rooms( room_id=room_id, @@ -595,17 +576,18 @@ def cmd_admin_link(room_id, pnut_chan_id): except pnutpy.errors.PnutAuthAPIException: errmsg = "- unable to subscribe to channel -" - logger.exception(errmsg) - return errmsg + logging.exception(errmsg) + return TextMessageEventContent(msgtype='m.text', body=errmsg) except Exception: errmsg = "- unable to link room for some reason -" - logger.exception(errmsg) - return errmsg + logging.exception(errmsg) + return TextMessageEventContent(msgtype='m.text', body=errmsg) -def cmd_admin_unlink(rid): - matrix_api = MatrixHttpApi(app.config['MATRIX_HOST'], - token=app.config['MATRIX_AS_TOKEN']) +async def cmd_admin_unlink(rid): + matrix_api = ClientAPI(app.config['MATRIX_AS_ID'], + base_url=app.config['MATRIX_HOST'], + token=app.config['MATRIX_AS_TOKEN']) pnutpy.api.add_authorization_token(app.config['MATRIX_PNUT_TOKEN']) if rid.startswith('!'): @@ -618,49 +600,62 @@ def cmd_admin_unlink(rid): alias = "#" + app.config['MATRIX_PNUT_PREFIX'] alias += str(room.pnut_chan) + ":" alias += app.config['MATRIX_DOMAIN'] - matrix_api.remove_room_alias(alias) + await matrix_api.remove_room_alias(alias) # Kicking users needs at least moderator privs - members = matrix_api.get_room_members(room.room_id) + members = await matrix_api.get_members(room.room_id) reason = "Portal room has been unlinked by administrator" for m in members['chunk']: - if m['content']['membership'] == 'join' and m['sender'] != app.config['MATRIX_AS_ID']: + if (m['content']['membership'] == 'join' and + m['sender'] != app.config['MATRIX_AS_ID']): if room.portal: - matrix_api.kick_user(room.room_id, m['sender'], reason=reason) + await matrix_api.kick_user(room.room_id, + m['sender'], + reason=reason) else: - if m['sender'].startswith("@" + app.config['MATRIX_PNUT_PREFIX']): - matrix_api.kick_user(room.room_id, m['sender'], reason=reason) + prefix = f"@{app.config['MATRIX_PNUT_PREFIX']}" + if m['sender'].startswith(prefix): + await matrix_api.kick_user(room.room_id, + m['sender'], + reason=reason) try: channel, meta = pnutpy.api.unsubscribe_channel(room.pnut_chan) - matrix_api.leave_room(room.room_id) + await matrix_api.leave_room(room.room_id) if room is not None: db_session.delete(room) db_session.commit() - return "- room has been unlinked -" + text = "- room has been unlinked -" else: - return "- unable to locate room to unlink -" + text = "- unable to locate room to unlink -" + + return TextMessageEventContent(msgtype='m.text', body=text) except Exception: errmsg = "- error while unlinking room -" - logger.exception(errmsg) - return errmsg + logging.exception(errmsg) + return TextMessageEventContent(msgtype='m.text', body=errmsg) -def on_direct_invite(event): +async def on_direct_invite(event): if event['state_key'] == app.config['MATRIX_AS_ID']: - matrix_api = MatrixHttpApi(app.config['MATRIX_HOST'], - token=app.config['MATRIX_AS_TOKEN']) + matrix_api = ClientAPI(app.config['MATRIX_AS_ID'], + base_url=app.config['MATRIX_HOST'], + token=app.config['MATRIX_AS_TOKEN']) dm = ControlRooms(room_id=event['room_id']) else: - matrix_api = MatrixHttpApi(app.config['MATRIX_HOST'], - identity=event['state_key'], - token=app.config['MATRIX_AS_TOKEN']) - bridge_user = event['state_key'] - pnut_user = bridge_user.replace(app.config['MATRIX_PNUT_PREFIX'],'').split(':')[0] + matrix_api = ClientAPI(app.config['MATRIX_AS_ID'], + base_url=app.config['MATRIX_HOST'], + token=app.config['MATRIX_AS_TOKEN'], + as_user_id=event['state_key']) - user = Users.query.filter(Users.matrix_id == event['sender']).one_or_none() + bridge_user = event['state_key'] + pnut_user = bridge_user.replace(app.config['MATRIX_PNUT_PREFIX'], + '').split(':')[0] + + user = Users.query.filter(Users.matrix_id == + event['sender']).one_or_none() if user is not None: # TODO: need to handle if the user isn't registered pnutpy.api.add_authorization_token(user.pnut_user_token) @@ -671,47 +666,48 @@ def on_direct_invite(event): bridge_user=bridge_user, pnut_chan=channel.id) try: - matrix_api.join_room(event['room_id']) + logging.debug('--> trying to join room <--') + await matrix_api.join_room_by_id(event['room_id']) db_session.add(dm) db_session.commit() except Exception: errmsg = "- on_direct_invite -" - logger.exception(errmsg) + logging.exception(errmsg) - return jsonify({}) - -def on_leave_event(event): - direct = DirectRooms.query.filter(DirectRooms.room_id == event['room_id']).one_or_none() +async def on_leave_event(event): + direct = DirectRooms.query.filter(DirectRooms.room_id == + event['room_id']).one_or_none() if direct is not None: + matrix_api = ClientAPI(app.config['MATRIX_AS_ID'], + base_url=app.config['MATRIX_HOST'], + token=app.config['MATRIX_AS_TOKEN'], + as_user_id=direct.bridge_user.lower()) - matrix_api = MatrixHttpApi(app.config['MATRIX_HOST'], - identity=direct.bridge_user, - token=app.config['MATRIX_AS_TOKEN']) try: - matrix_api.leave_room(event['room_id']) + await matrix_api.leave_room(event['room_id']) db_session.delete(direct) db_session.commit() except Exception: errmsg = "- on_leave_event -" - logger.exception(errmsg) + logging.exception(errmsg) - control = ControlRooms.query.filter(ControlRooms.room_id == event['room_id']).one_or_none() + control = ControlRooms.query.filter(ControlRooms.room_id == + event['room_id']).one_or_none() if control is not None: - matrix_api = MatrixHttpApi(app.config['MATRIX_HOST'], - token=app.config['MATRIX_AS_TOKEN']) + matrix_api = ClientAPI(app.config['MATRIX_AS_ID'], + base_url=app.config['MATRIX_HOST'], + token=app.config['MATRIX_AS_TOKEN']) try: - matrix_api.leave_room(event['room_id']) + await matrix_api.leave_room(event['room_id']) db_session.delete(control) db_session.commit() except Exception: errmsg = "- on_leave_event -" - logger.exception(errmsg) - - return jsonify({}) + logging.exception(errmsg) def on_direct_message(event, user, room): if user is not None: @@ -719,20 +715,21 @@ def on_direct_message(event, user, room): prefix = "" else: token = app.config['MATRIX_PNUT_TOKEN'] - matrix_profile = get_profile(event['user_id']) + matrix_profile = get_profile(event['sender']) if "displayname" in matrix_profile: - prefix = "[" + matrix_profile['displayname'] + "] (" + event['user_id'] + ")\n" + prefix = (f"[{matrix_profile['displayname']}]" + f" ({event['sender']})\n") else: - prefix = "(" + event['user_id'] + ")\n" + prefix = "(" + event['sender'] + ")\n" pnutpy.api.add_authorization_token(token) - embed = [crosspost_raw(event)] + raw = {} + raw['io.pnut.core.crosspost'] = [crosspost_raw(event)] evtext, evraw = msg_from_event(event) text = prefix + evtext - if len(evraw) > 0: - embed.append(evraw) try: - msg, meta = pnutpy.api.create_message(room.pnut_chan, data={'text': text, 'raw': embed}) + msg, meta = pnutpy.api.create_message(room.pnut_chan, + data={'text': text, 'raw': raw}) revent = Events( event_id=event['event_id'], room_id=event['room_id'], @@ -745,17 +742,18 @@ def on_direct_message(event, user, room): db_session.commit() except pnutpy.errors.PnutAuthAPIException: - logger.exception('-unable to post to pnut channel-') + logging.exception('-unable to post to pnut channel-') except Exception: - logger.exception('-something bad happened here-') + logging.exception('-something bad happened here-') return jsonify({}) -def on_control_message(event): - matrix_api = MatrixHttpApi(app.config['MATRIX_HOST'], - token=app.config['MATRIX_AS_TOKEN']) - logger.debug("- direct room event received -") +async def on_control_message(event): + matrix_api = ClientAPI(app.config['MATRIX_AS_ID'], + base_url=app.config['MATRIX_HOST'], + token=app.config['MATRIX_AS_TOKEN']) + logging.debug("- direct room event received -") if event['type'] != 'm.room.message': return jsonify({}) @@ -765,37 +763,43 @@ def on_control_message(event): try: if msg[0] == '!help' or msg[0] == 'help': if len(msg) > 1: - matrix_api.send_message(event['room_id'], cmd_user_help(msg[1])) + await matrix_api.send_message(event['room_id'], + cmd_user_help(msg[1])) else: - matrix_api.send_message(event['room_id'], cmd_user_help()) + await matrix_api.send_message(event['room_id'], + cmd_user_help()) elif msg[0] == '!auth': - matrix_api.send_message(event['room_id'], cmd_user_auth()) + await matrix_api.send_message(event['room_id'], cmd_user_auth()) elif msg[0] == '!save': if len(msg) > 1: - matrix_api.send_message(event['room_id'], cmd_user_save(event['sender'], msg[1])) + await matrix_api.send_message(event['room_id'], + cmd_user_save(event['sender'], + msg[1])) else: - matrix_api.send_message(event['room_id'], cmd_user_save()) + await matrix_api.send_message(event['room_id'], + cmd_user_save()) elif msg[0] == '!drop': - matrix_api.send_message(event['room_id'], cmd_user_drop(event['sender'])) + await matrix_api.send_message(event['room_id'], + cmd_user_drop(event['sender'])) elif msg[0] == '!status': - matrix_api.send_message(event['room_id'], cmd_user_status(event['sender'])) + await matrix_api.send_message(event['room_id'], + cmd_user_status(event['sender'])) elif msg[0] == '!join': if len(msg) > 1: - matrix_api.send_message(event['room_id'], cmd_user_join(event['sender'], msg[1])) + r = await cmd_user_join(event['sender'], msg[1]) + await matrix_api.send_message(event['room_id'], r) else: - matrix_api.send_message(event['room_id'], cmd_user_join(event['sender'])) - + r = await cmd_user_join(event['sender']) + await matrix_api.send_message(event['room_id'], r) except Exception: errmsg = "- on_direct_message -" - logger.exception(errmsg) - - return jsonify({}) + logging.exception(errmsg) def cmd_user_help(cmd=None): reply = "This is an admin room for controlling your connection to pnut.io\n" @@ -806,7 +810,7 @@ def cmd_user_help(cmd=None): reply += "!status\t\t\t- Status of your pnut.io auth token\n" reply += "!join \t- Join a private channel on pnut.io\n" - return reply + return TextMessageEventContent(msgtype='m.text', body=reply) def cmd_user_auth(): reply = "Visit the following URL to authorize your account on pnut.io.\n\n" @@ -816,13 +820,13 @@ def cmd_user_auth(): reply += "&scope=write_post,presence,messages&response_type=token\n\n" reply += "You will be provided with a token that you store with the !save command.\n" - return reply + return TextMessageEventContent(msgtype='m.text', body=reply) def cmd_user_save(sender=None, token=None): if token is None: reply = "You must provide a token with this command.\n" reply += "!save " - return reply + return TextMessageEventContent(msgtype='m.text', body=reply) pnutpy.api.add_authorization_token(token) try: @@ -845,7 +849,7 @@ def cmd_user_save(sender=None, token=None): logging.exception('!save') reply = "Error! Problem saving your token." - return reply + return TextMessageEventContent(msgtype='m.text', body=reply) def cmd_user_drop(sender=None): try: @@ -861,7 +865,7 @@ def cmd_user_drop(sender=None): logging.exception('!drop') reply = "Error! Problem removing your token." - return reply + return TextMessageEventContent(msgtype='m.text', body=reply) def cmd_user_status(sender=None): try: @@ -880,13 +884,13 @@ def cmd_user_status(sender=None): logging.exception('!status') reply = "Error! There was a problem checking your account." - return reply + return TextMessageEventContent(msgtype='m.text', body=reply) -def cmd_user_join(sender=None, channel_id=None): +async def cmd_user_join(sender=None, channel_id=None): if channel_id is None: reply = "You must provide a channel id number with this command.\n" reply += "!join " - return reply + return TextMessageEventContent(msgtype='m.text', body=reply) try: user = Users.query.filter(Users.matrix_id == sender).one_or_none() @@ -895,13 +899,15 @@ def cmd_user_join(sender=None, channel_id=None): else: pnutpy.api.add_authorization_token(user.pnut_user_token) channel, meta = pnutpy.api.get_channel(channel_id, include_raw=1) - room = Rooms.query.filter(Rooms.pnut_chan == channel_id).one_or_none() + room = Rooms.query.filter(Rooms.pnut_chan == + channel_id).one_or_none() if room is None: - create_room(channel, user) + await create_pnut_matrix_room(channel, user) else: - matrix_api = MatrixHttpApi(app.config['MATRIX_HOST'], - token=app.config['MATRIX_AS_TOKEN']) - matrix_api.invite_user(room.room_id, sender) + matrix_api = ClientAPI(app.config['MATRIX_AS_ID'], + base_url=app.config['MATRIX_HOST'], + token=app.config['MATRIX_AS_TOKEN']) + await matrix_api.invite_user(room.room_id, sender) reply = "ok" except pnutpy.errors.PnutAuthAPIException as e: @@ -914,4 +920,36 @@ def cmd_user_join(sender=None, channel_id=None): logging.exception('!join') reply = "Error! There was a problem joining the channel." - return reply + return TextMessageEventContent(msgtype='m.text', body=reply) + +class MLogFilter(logging.Filter): + + ACCESS_TOKEN_RE = re.compile(r"(\?.*access(_|%5[Ff])token=)[^&]*(\s.*)$") + ACCESS_TOKEN_RE2 = re.compile(r"(\?.*access(_|%5[Ff])token=)[^&]*(.*)$") + + def filter(self, record): + if record.name == "werkzeug" and len(record.args) > 0: + redacted_uri = MLogFilter.ACCESS_TOKEN_RE.sub(r"\1\3", + record.args[0]) + record.args = (redacted_uri, ) + record.args[1:] + elif record.name == "urllib3.connectionpool" and len(record.args) > 3: + redacted_uri = MLogFilter.ACCESS_TOKEN_RE2.sub(r"\1\3", + record.args[4]) + record.args = record.args[:4] + (redacted_uri,) + record.args[5:] + + return True + +if __name__ == '__main__': + configyaml = os.environ.get("CONFIG_FILE") + + with open(configyaml, "rb") as config_file: + config = yaml.load(config_file, Loader=yaml.SafeLoader) + + logging.config.dictConfig(config['logging']) + redact_filter = MLogFilter() + logging.getLogger("werkzeug").addFilter(redact_filter) + logging.getLogger("urllib3.connectionpool").addFilter(redact_filter) + + app.config.update(config) + logging.basicConfig(level=logging.DEBUG) + app.run(host=config['LISTEN_HOST'], port=config['LISTEN_PORT']) diff --git a/pnut-matrix.py b/pnut-matrix.py index 993f821..8dbc6ac 100644 --- a/pnut-matrix.py +++ b/pnut-matrix.py @@ -1,5 +1,3 @@ -import websocket -import threading import time import logging import logging.config @@ -11,19 +9,22 @@ import magic import argparse import os import re +import asyncio + +from mautrix.client import ClientAPI +from mautrix.types import TextMessageEventContent, Format, MessageType +from mautrix.errors import MatrixConnectionError +from mautrix.errors.request import MNotFound, MForbidden + +from websockets.asyncio.client import connect +from websockets.exceptions import ConnectionClosed -from matrix_client.api import MatrixHttpApi -from matrix_client.api import MatrixError, MatrixRequestError from models import Avatars, Rooms, Events, DirectRooms, Users from database import db_session, init_db from sqlalchemy import and_ -from appservice import app logger = logging.getLogger() -_shutdown = threading.Event() -_reconnect = threading.Event() - class MLogFilter(logging.Filter): ACCESS_TOKEN_RE = re.compile(r"(\?.*access(_|%5[Ff])token=)[^&]*(\s.*)$") @@ -31,15 +32,17 @@ class MLogFilter(logging.Filter): def filter(self, record): if record.name == "werkzeug" and len(record.args) > 0: - redacted_uri = MLogFilter.ACCESS_TOKEN_RE.sub(r"\1\3", record.args[0]) + redacted_uri = MLogFilter.ACCESS_TOKEN_RE.sub(r"\1\3", + record.args[0]) record.args = (redacted_uri, ) + record.args[1:] elif record.name == "urllib3.connectionpool" and len(record.args) > 3: - redacted_uri = MLogFilter.ACCESS_TOKEN_RE2.sub(r"\1\3", record.args[4]) + redacted_uri = MLogFilter.ACCESS_TOKEN_RE2.sub(r"\1\3", + record.args[4]) record.args = record.args[:4] + (redacted_uri,) + record.args[5:] return True -def new_message(msg, meta): +async def new_pnut_message(msg, meta): logger.debug("channel: " + msg.channel_id) logger.debug("username: " + msg.user.username) if 'name' in msg.user: @@ -53,10 +56,18 @@ def new_message(msg, meta): if msg.source.id == config['PNUTCLIENT_ID']: return + matrix_id = matrix_id_from_pnut(msg.user.username) + matrix_api = ClientAPI(config['MATRIX_AS_ID'], + base_url=config['MATRIX_HOST'], + token=config['MATRIX_AS_TOKEN'], + as_user_id=matrix_id.lower()) + if meta['channel_type'] == 'io.pnut.core.chat': - room = Rooms.query.filter(Rooms.pnut_chan == msg.channel_id).one_or_none() + room = Rooms.query.filter(Rooms.pnut_chan == + msg.channel_id).one_or_none() elif meta['channel_type'] == 'io.pnut.core.pm': - room = DirectRooms.query.filter(DirectRooms.pnut_chan == msg.channel_id).one_or_none() + room = DirectRooms.query.filter(DirectRooms.pnut_chan == + msg.channel_id).one_or_none() if room is None: # Do do an invite from the bridge user? logger.debug('new invite?') @@ -64,12 +75,13 @@ def new_message(msg, meta): # subscribed_user_ids from meta logger.debug(meta['subscribed_user_ids']) pnut_user = matrix_id_from_pnut(msg.user.username) - profile = get_matrix_profile(pnut_user) + profile = await matrix_api.get_profile(matrix_id.lower()) if not profile: new_matrix_user(msg.user.username) invitees=[] for pm_user in meta['subscribed_user_ids']: - user = Users.query.filter(Users.pnut_user_id == pm_user).one_or_none() + user = Users.query.filter(Users.pnut_user_id == + pm_user).one_or_none() if int(pm_user) == msg.user.id: continue if user is not None: @@ -84,49 +96,39 @@ def new_message(msg, meta): logger.debug('-not_mapped-') return - matrix_id = matrix_id_from_pnut(msg.user.username) - matrix_api = MatrixHttpApi(config['MATRIX_HOST'], - token=config['MATRIX_AS_TOKEN'], - identity=matrix_id) + try: + profile = await matrix_api.get_profile(matrix_id.lower()) + logger.debug(profile) - profile = get_matrix_profile(matrix_id) - if not profile: + except MNotFound: new_matrix_user(msg.user.username) logger.debug('-new_user-') profile = {'displayname': None} if profile['displayname'] != matrix_display_from_pnut(msg.user): - set_matrix_display(msg.user) + await set_matrix_display(msg.user) logger.debug('-set_display-') - avatar = Avatars.query.filter(Avatars.pnut_user == msg.user.username).one_or_none() + avatar = Avatars.query.filter(Avatars.pnut_user == + msg.user.username).one_or_none() if avatar is None or avatar.avatar != msg.user.content.avatar_image.url: - set_matrix_avatar(msg.user) + await set_matrix_avatar(msg.user) logger.debug('-set_avatar-') # members = matrix_api.get_room_members(room.room_id) # logger.debug(members) # join_room(room.room_id, config['MATRIX_AS_ID']) # TODO: sort out room invite and join logic - join_room(room.room_id, matrix_id) + await join_room(room.room_id, matrix_id) if 'content' in msg: - text = msg.content.text + "\n" - ts = int(time.time()) * 1000 - - lnktext = "" - for link in msg.content.entities.links: - if 'title' in link: - lnktext += link.title + "\n" - if 'url' in link: - lnktext += link.url + "\n" - - if len(lnktext) > 0: - text += "\n" + lnktext - - r = matrix_api.send_message(room.room_id, text, timestamp=ts) + eventtext = TextMessageEventContent(msgtype=MessageType.TEXT, + format=Format.HTML, + body=msg.content.text, + formatted_body=msg.content.html) + rid = await matrix_api.send_message(room.room_id, eventtext) event = Events( - event_id=r['event_id'], + event_id=rid, room_id=room.room_id, pnut_msg_id=msg.id, pnut_user_id=msg.user.id, @@ -137,14 +139,80 @@ def new_message(msg, meta): if 'raw' in msg: logger.debug('-handle media uploads-') - new_media(room.room_id, msg) + await new_media(room.room_id, msg) -def new_media(room_id, msg): +async def new_pnut_post(post, meta): + + if not config['PNUT_GLOBAL']: + return + + if (config['PNUT_GLOBAL_HUMAN_ONLY'] and + post.user.type in ['feed', 'bot']): + logging.debug('-skipping non human post-') + return + + if 'content' in post: + + text = "" + if 'repost_of' in post: + text += f"<{post.user.username}> reposted >> " + post = post.repost_of + + matrix_id = matrix_id_from_pnut(post.user.username) + matrix_api = ClientAPI(config['MATRIX_AS_ID'], + base_url=config['MATRIX_HOST'], + token=config['MATRIX_AS_TOKEN'], + as_user_id=matrix_id.lower()) + try: + profile = await matrix_api.get_profile(matrix_id.lower()) + + except MNotFound: + new_matrix_user(post.user.username) + + profile = {'displayname': None} + + if profile['displayname'] != matrix_display_from_pnut(post.user): + await set_matrix_display(post.user) + logger.debug('-set_display-') + + avatar = Avatars.query.filter(Avatars.pnut_user == + post.user.username).one_or_none() + if (avatar is None or + avatar.avatar != post.user.content.avatar_image.url): + await set_matrix_avatar(post.user) + logger.debug('-set_avatar-') + + room_id = config['MATRIX_GLOBAL_ROOM'] + await join_room(room_id, matrix_id) + postlink = f"https://posts.pnut.io/{post.id}" + plaintext = f"{post.content.text}\n{postlink}" + htmltext = (f"{post.content.html}" + f"  [🔗]") + eventtext = TextMessageEventContent(msgtype=MessageType.TEXT, + format=Format.HTML, + body=plaintext, + formatted_body=htmltext) + rid = await matrix_api.send_message(room_id, eventtext) + event = Events( + event_id=rid, + room_id=room_id, + pnut_msg_id=post.id, + pnut_user_id=post.user.id, + pnut_chan_id=0, + deleted=False) + db_session.add(event) + db_session.commit() + + if 'raw' in post: + logger.debug('-handle media uploads-') + await new_media(room_id, post) + +async def new_media(room_id, msg): matrix_id = matrix_id_from_pnut(msg.user.username) - matrix_api = MatrixHttpApi(config['MATRIX_HOST'], - token=config['MATRIX_AS_TOKEN'], - identity=matrix_id) - ts = int(time.time()) * 1000 + matrix_api = ClientAPI(config['MATRIX_AS_ID'], + base_url=config['MATRIX_HOST'], + token=config['MATRIX_AS_TOKEN'], + as_user_id=matrix_id.lower()) if 'io.pnut.core.oembed' in msg.raw: @@ -157,16 +225,20 @@ def new_media(room_id, msg): info['h'] = oembed.height info['w'] = oembed.width elif oembed.type == 'audio': - logger.debug("* recieved audio attachment") - continue + msgtype = 'm.audio' + dl_url = oembed.url elif oembed.type == 'video': - logger.debug("* recieved video attachment") - continue + msgtype = 'm.video' + dl_url = oembed.url + info['h'] = oembed.height + info['w'] = oembed.width elif oembed.type == 'html5video': - logger.debug("* recieved html5 video attachment") - continue + msgtype = 'm.video' + dl_url = oembed.url + info['h'] = oembed.height + info['w'] = oembed.width elif oembed.type == 'rich': - logger.debug("* recieved video attachment") + logger.debug("* recieved rich attachment") continue else: logger.debug("* recieved unknown attachment") @@ -177,128 +249,178 @@ def new_media(room_id, msg): with magic.Magic(flags=magic.MAGIC_MIME_TYPE) as m: info['mimetype'] = m.id_buffer(dl.content) info['size'] = len(dl.content) - ul = matrix_api.media_upload(dl.content, info['mimetype']) + ul = await matrix_api.upload_media(dl.content, + mime_type=info['mimetype']) if 'title' in oembed: title = oembed.title else: title = "" - r = matrix_api.send_content(room_id, ul['content_uri'], title, msgtype, extra_information=info, timestamp=ts) + rid = await matrix_api.send_file(room_id, ul, + file_name=title, + file_type=msgtype, + info=info) + if 'channel_id' in msg: + channel_id = msg.channel_id + else: + channel_id = 0 event = Events( - event_id=r['event_id'], + event_id=rid, room_id=room_id, pnut_msg_id=msg.id, pnut_user_id=msg.user.id, - pnut_chan_id=msg.channel_id, + pnut_chan_id=channel_id, deleted=False) db_session.add(event) db_session.commit() -def delete_message(msg): +async def delete_message(msg): matrix_id = matrix_id_from_pnut(msg.user.username) - matrix_api = MatrixHttpApi(config['MATRIX_HOST'], - token=config['MATRIX_AS_TOKEN'], - identity=matrix_id) + matrix_api = ClientAPI(config['MATRIX_AS_ID'], + base_url=config['MATRIX_HOST'], + token=config['MATRIX_AS_TOKEN'], + as_user_id=matrix_id.lower()) - events = Events.query.filter(and_(Events.pnut_msg_id == msg.id, Events.deleted == False)).all() + events = Events.query.filter(and_(Events.pnut_msg_id == + msg.id, Events.deleted == False)).all() for event in events: - matrix_api.redact_event(event.room_id, event.event_id) + await matrix_api.redact(event.room_id, event.event_id) event.deleted = True db_session.commit() def matrix_id_from_pnut(username): - return "@" + config['MATRIX_PNUT_PREFIX'] + username + ":" + config['MATRIX_DOMAIN'] + matrix_id = (f"@{config['MATRIX_PNUT_PREFIX']}{username}" + f":{config['MATRIX_DOMAIN']}") + return matrix_id def matrix_display_from_pnut(user): + if user.type == 'bot': + icon = ' 🤖' + elif user.type == 'feed': + icon = ' 📰' + else: + icon = '' + if 'name' in user: - display = user.name + " (@" + user.username + ")" + display = user.name + " (@" + user.username + ")" + icon else: - display = "@" + user.username + display = "@" + user.username + icon return display - # return user.username + " (pnut)" -def get_matrix_profile(matrix_id): - url = matrix_url + '/profile/' + matrix_id - r = requests.get(url) - - if r.status_code == 200: - return r.json() - else: - return None - -def set_matrix_display(user): +async def set_matrix_display(user): matrix_id = matrix_id_from_pnut(user.username) - matrix_api = MatrixHttpApi(config['MATRIX_HOST'], - token=config['MATRIX_AS_TOKEN'], - identity=matrix_id) - matrix_api.set_display_name(matrix_id, matrix_display_from_pnut(user)) + matrix_api = ClientAPI(config['MATRIX_AS_ID'], + base_url=config['MATRIX_HOST'], + token=config['MATRIX_AS_TOKEN'], + as_user_id=matrix_id.lower()) + await matrix_api.set_displayname(matrix_display_from_pnut(user)) -def set_matrix_avatar(user): +async def set_matrix_avatar(user): matrix_id = matrix_id_from_pnut(user.username) - matrix_api = MatrixHttpApi(config['MATRIX_HOST'], - token=config['MATRIX_AS_TOKEN'], - identity=matrix_id) + matrix_api = ClientAPI(config['MATRIX_AS_ID'], + base_url=config['MATRIX_HOST'], + token=config['MATRIX_AS_TOKEN'], + as_user_id=matrix_id.lower()) dl = requests.get(user.content.avatar_image.url, stream=True) dl.raise_for_status() with magic.Magic(flags=magic.MAGIC_MIME_TYPE) as m: mtype = m.id_buffer(dl.content) - ul = matrix_api.media_upload(dl.content, mtype) + ul = await matrix_api.upload_media(dl.content, mtype) + logger.debug(ul) try: - matrix_api.set_avatar_url(matrix_id, ul['content_uri']) - avatar = Avatars.query.filter(Avatars.pnut_user == user.username).one_or_none() + await matrix_api.set_avatar_url(ul) + avatar = Avatars.query.filter(Avatars.pnut_user == + user.username).one_or_none() if avatar is None: - avatar = Avatars(pnut_user=user.username, avatar=user.content.avatar_image.url) + avatar = Avatars(pnut_user=user.username, + avatar=user.content.avatar_image.url) db_session.add(avatar) else: avatar.avatar = user.content.avatar_image.url db_session.commit() - except MatrixRequestError: + except Exception: logger.exception('failed to set user avatar') def new_matrix_user(username): - matrix_api = MatrixHttpApi(config['MATRIX_HOST'], - token=config['MATRIX_AS_TOKEN']) + endpoint = "/_matrix/client/v3/register" + url = config['MATRIX_HOST'] + endpoint + params = {'kind': 'user'} data = { 'type': 'm.login.application_service', 'username': config['MATRIX_PNUT_PREFIX'] + username } - matrix_api.register(content=data) + headers = { + "Content-Type": "application/json", + "Authorization": "Bearer " + config['MATRIX_AS_TOKEN'] + } + logger.debug(data) + r = requests.post(url, headers=headers, json=data, params=params) + if r.status_code == 200: + return -def join_room(room_id, matrix_id): - matrix_api_as = MatrixHttpApi(config['MATRIX_HOST'], - token=config['MATRIX_AS_TOKEN']) - matrix_api = MatrixHttpApi(config['MATRIX_HOST'], - token=config['MATRIX_AS_TOKEN'], - identity=matrix_id) + else: + errmsg = f"- unable to register {username} -" + logger.warning(errmsg) + logger.debug(r.status_code) + logger.debug(r.text) + return + +async def join_room(room_id, matrix_id): + logging.debug('----- trying to join room -----') + + matrix_api_as = ClientAPI(config['MATRIX_AS_ID'], + base_url=config['MATRIX_HOST'], + token=config['MATRIX_AS_TOKEN']) + + matrix_api = ClientAPI(config['MATRIX_AS_ID'], + base_url=config['MATRIX_HOST'], + token=config['MATRIX_AS_TOKEN'], + as_user_id=matrix_id.lower()) try: - matrix_api.join_room(room_id) + await matrix_api.join_room(room_id) + # logging.debug('----- should be joined -----') - except MatrixRequestError as e: - if e.code == 403: - matrix_api_as.invite_user(room_id, matrix_id) - matrix_api.join_room(room_id) - else: - logger.exception('failed to join room') + except MForbidden: + # logging.debug('------ got a forbidden ------') + await matrix_api_as.invite_user(room_id, matrix_id.lower()) + await matrix_api.join_room(room_id) + + except MatrixConnectionError: + # logger.debug(e) + # if 'code' in e and e.code == 403: + # await matrix_api_as.invite_user(room_id, matrix_id) + # await matrix_api.join_room(room_id) + # else: + logging.debug('------- moar join errors -------') + logger.exception('failed to join room') + logger.debug(f"{room_id}") logger.debug('-room_join-') def new_room(pnut_user, invitees, chan): dr = None - matrix_api = MatrixHttpApi(config['MATRIX_HOST'], - token=config['MATRIX_AS_TOKEN'], - identity=pnut_user) url = matrix_url + '/createRoom' - params = {"access_token": config['MATRIX_AS_TOKEN'], "user_id": pnut_user} - content = {"visibility": "private", "is_direct": True, "invite": invitees} + params = { + "access_token": config['MATRIX_AS_TOKEN'], + "user_id": pnut_user.lower() + } + content = { + "visibility": "private", + "is_direct": True, + "invite": invitees + } headers = {"Content-Type": "application/json"} r = requests.post(url, headers=headers, params=params, data=json.dumps(content)) response = r.json() + logger.debug(r.status_code) + logger.debug(r.text) + logger.debug(response) for bridge_user in invitees: dr = DirectRooms(room_id=response['room_id'], bridge_user=pnut_user, pnut_chan=chan) @@ -308,83 +430,71 @@ def new_room(pnut_user, invitees, chan): return dr -def on_message(ws, message): - # logger.debug("on_message: " + message) +async def on_message(message): + logger.debug("on_message: " + message) msg = json.loads(message) - logger.debug(msg['meta']) + + if 'meta' in msg: + meta = msg['meta'] + else: + return if 'data' in msg: + data = msg['data'] + else: + return - if 'channel_type' in msg['meta']: + if 'type' in meta: - if msg['meta']['channel_type'] not in ['io.pnut.core.chat', - 'io.pnut.core.pm']: + if meta['type'] == "message": + + channel_types = ['io.pnut.core.chat', 'io.pnut.core.pm'] + if meta['channel_type'] not in channel_types: return - for d_item in msg['data']: - pmsg = pnutpy.models.Message.from_response_data(d_item) + for item in data: + pnut_msg = pnutpy.models.Message.from_response_data(item) + + if 'is_deleted' in meta and meta['is_deleted']: + logger.debug("-message: delete-") + delete_message(pnut_msg) - if 'is_deleted' in msg['meta']: - if msg['meta']['is_deleted']: - logger.debug("message: delete") - delete_message(pmsg) else: - new_message(pmsg, msg['meta']) + await new_pnut_message(pnut_msg, meta) -def on_error(ws, error): - logger.error("on_error: !!! ERROR !!!") - logger.error(error) + elif meta['type'] == "post": -def on_close(ws): - logger.debug("on_close: ### CLOSED ###") + for item in data: + pnut_post = pnutpy.models.Post.from_response_data(item) + await new_pnut_post(pnut_post, meta) -def on_open(ws): +async def asmain(): + if config['MATRIX_ADMIN_ROOM']: + logger.debug("- sould join admin room -") + await join_room(config['MATRIX_ADMIN_ROOM'], config['MATRIX_AS_ID']) - def run(*args): - while not _shutdown.isSet() and not _reconnect.isSet(): - time.sleep(3) - try: - ws.send(".") - except websocket._exceptions.WebSocketConnectionClosedException: - logger.debug('websocket closed exception caught...') - _reconnect.set() + ws_url = 'wss://stream.pnut.io/v1/app?access_token=' + ws_url += config['PNUT_APPTOKEN'] + '&key=' + config['PNUT_APPKEY'] + ws_url += '&include_raw=1' + async for websocket in connect(uri=ws_url): + try: + async for message in websocket: + await on_message(message) - time.sleep(1) - logger.debug("*** terminate thread ***") + await websocket.close() - t = threading.Thread(target=run) - t.start() - -def wsthreader(threadfunc): - - def wrapper(): - - while not _shutdown.isSet(): - _reconnect.clear() - logger.debug('threadfunc start...') - running = threadfunc() - logger.debug('threadfunc end...') - if running: - time.sleep(5) - else: - _shutdown.set() - logger.debug('*** thread stopped ***') - - return wrapper + except ConnectionClosed: + continue if __name__ == '__main__': - a_parser = argparse.ArgumentParser() a_parser.add_argument( '-d', action='store_true', dest='debug', help="debug logging" ) - # TODO: solve the database.py problem and enable this - # a_parser.add_argument( - # '-c', '--config', default="config.yaml", - # help="configuration file" - # ) + a_parser.add_argument('-c', '--config', dest='configyaml', + default="config.yaml", help="configuration file") args = a_parser.parse_args() configyaml = os.environ.get("CONFIG_FILE") @@ -392,35 +502,16 @@ if __name__ == '__main__': with open(configyaml, "rb") as config_file: config = yaml.load(config_file, Loader=yaml.SafeLoader) - # websocket.enableTrace(True) logging.config.dictConfig(config['logging']) redact_filter = MLogFilter() logging.getLogger("werkzeug").addFilter(redact_filter) logging.getLogger("urllib3.connectionpool").addFilter(redact_filter) - ws_url = 'wss://stream.pnut.io/v1/app?access_token=' - ws_url += config['PNUT_APPTOKEN'] + '&key=' + config['PNUT_APPKEY'] - ws_url += '&include_raw=1' - matrix_url = config['MATRIX_HOST'] + '/_matrix/client/r0' + matrix_url = config['MATRIX_HOST'] + '/_matrix/client/v3' # setup the database connection init_db() - # setup the websocket connection - ws = websocket.WebSocketApp(ws_url, on_message=on_message, - on_error=on_error, on_close=on_close) - ws.on_open = on_open - wst = threading.Thread(target=wsthreader(ws.run_forever)) - wst.start() - - # setup the matrix app service - if config['MATRIX_ADMIN_ROOM']: - logger.debug("- sould join admin room -") - join_room(config['MATRIX_ADMIN_ROOM'], config['MATRIX_AS_ID']) - app.config.update(config) - app.run(host=config['LISTEN_HOST'], port=config['LISTEN_PORT']) + asyncio.run(asmain()) logger.info('!! shutdown initiated !!') - _shutdown.set() - ws.close() - time.sleep(2) diff --git a/requirements.txt b/requirements.txt index ae1f4cf..bfbd87f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,9 @@ pyyaml requests -matrix-client==0.3.2 -Flask -pnutpy +Flask[async] +pnutpy>=0.5.0 sqlalchemy -websocket-client filemagic +mautrix>=0.20.6,<0.21 +websockets +asyncclick