Sort and group a list of dictionaries
<p>How can I sort and group this list of dictionaries into a nested dictionary which I want to return via an API as JSON.</p>

<p>Source Data (list of permissions):</p>

<pre><code>[{
    'can_create': True,
    'can_read': True,
    'module_name': 'ModuleOne',
    'module_id': 1,
    'role_id': 1,
    'end_point_id': 1,
    'can_update': True,
    'end_point_name': 'entity',
    'can_delete': True,
}, {
    'can_create': True,
    'can_read': True,
    'module_name': 'ModuleTwo',
    'module_id': 2,
    'role_id': 1,
    'end_point_id': 4,
    'can_update': True,
    'end_point_name': 'financial-outlay',
    'can_delete': True,
},{
    'can_create': True,
    'can_read': True,
    'module_name': 'ModuleOne',
    'module_id': 1,
    'role_id': 1,
    'end_point_id': 2,
    'can_update': True,
    'end_point_name': 'management-type',
    'can_delete': True,
}, {
    'can_create': True,
    'can_read': True,
    'module_name': 'ModuleOne',
    'module_id': 1,
    'role_id': 1,
    'end_point_id': 3,
    'can_update': True,
    'end_point_name': 'ownership-type',
    'can_delete': False,
}, {
    'can_create': True,
    'can_read': True,
    'module_name': 'ModuleTwo',
    'module_id': 2,
    'role_id': 1,
    'end_point_id': 5,
    'can_update': True,
    'end_point_name': 'exposure',
    'can_delete': True,
}]
</code></pre>

<p>I want to transform that into a nested dicitonary object for return with an API as JSON. Here's the expected output:</p>

<pre><code>{
    "role_id": 1,
    "modules": [{
            "module_id": 1,
            "module_name": "ModuleOne",
            "permissions": [{
                "end_point_id": 1,
                "end_point_name": "entity",
                "can_create": False,
                "can_read": True,
                "can_write": True,
                "can_delete": True
            }, {
                "end_point_id": 2,
                "end_point_name": "management-type",
                "can_create": False,
                "can_read": True,
                "can_write": True,
                "can_delete": True
            }, {
                "end_point_id": 3,
                "end_point_name": "ownership-type",
                "can_create": False,
                "can_read": True,
                "can_write": True,
                "can_delete": True
            }, ]
        }, {
            "module_id": 2,
            "module_name": "ModuleTwo",
            "permissions": [{
                "end_point_id": 4,
                "end_point_name": "financial-outlay",
                "can_create": False,
                "can_read": True,
                "can_write": True,
                "can_delete": True
            }, {
                "end_point_id": 5,
                "end_point_name": "exposure",
                "can_create": False,
                "can_read": True,
                "can_write": True,
                "can_delete": True
            }, ]
        },

    ]
}
</code></pre>

<p>It looked trivial until I spent more time that I would expect trying to bend my mind around it. I've attempted so many options with non of them working. Here's the last attempt.</p>

<pre><code># Get user role
user_roles = get_user_roles()  # List of roles e.g. [{'role_id':1, role_name: 'role_one'}, {'role_id':2, role_name: 'role_two'}]

for role in user_roles:
    role_id = role['role_id']
    role_name = role['role_name']

    # Fetch Role Permissions
    role_permissions = get_role_permissions(role_id)  # List of permissions as seen above
    sorted_role_permissions = sorted(role_permissions, key=itemgetter('module_id'))  # sort dictionaries in list by 'module_id'

    modules_list = []
    permissions_list = []
    previous_module_id = 0
    is_first_loop = True
    for role_permission in sorted_role_permissions:

        module_id = role_permission['module_id']
        module_name = role_permission['module_name']
        end_point_id = role_permission['end_point_id']
        end_point_name = role_permission['end_point_name']

        if is_first_loop:
            print(0)
            is_first_loop = False
            previous_module_id = module_id
            print('end_point_name 0 {}'.format(end_point_name))
            permissions = {'end_point_id': end_point_id, 'end_point_name': end_point_name,
                           'can_create': role_permission['can_create'],
                           'can_read': role_permission['can_read'],
                           'can_update': role_permission['can_update'],
                           'can_delete': role_permission['can_delete']
                           }
            permissions_list.append(permissions)
            if len(sorted_role_permissions) == 1:
                # If there is only one permission in the role, end the loop
                modules_dict = {'module_id': module_id, 'module_name': module_name,
                                'permissions': permissions_list}
                modules_list.append(modules_dict)
                break

        else:

            if module_id == previous_module_id:
                # As long as the current module_id and the previous_module_id are the same, add to the same list

                print(1)
                permissions = {'end_point_id': end_point_id, 'end_point_name': end_point_name,
                               'can_create': role_permission['can_create'],
                               'can_read': role_permission['can_read'],
                               'can_update': role_permission['can_update'],
                               'can_delete': role_permission['can_delete']
                               }
                permissions_list.append(permissions)

            else:

                print(2)
                modules_dict = {'module_id': module_id, 'module_name': module_name,
                                'permissions': permissions_list}
                modules_list.append(modules_dict)
                permissions_list = []
                permissions = {'end_point_id': end_point_id, 'end_point_name': end_point_name,
                               'can_create': role_permission['can_create'],
                               'can_read': role_permission['can_read'],
                               'can_update': role_permission['can_update'],
                               'can_delete': role_permission['can_delete']
                               }
                permissions_list.append(permissions)
                previous_module_id = module_id

    if modules_list:
        roles.append({'role_id': role_id, 'role_name': role_name, 'modules': modules_list})
</code></pre>

<p>PyFunctional is pretty good at list manipulation.</p>

<pre><code>from pprint import pprint
from functional import seq
input = [...]  # taken from your example
output =(
    seq(input)  # convert regular python list to Sequence object

    # group by role_id
    .map(lambda e: (e.pop('role_id'), e)).group_by_key()

    # start building role dict
    .map(lambda role_modules: {
        "role_id": role_modules[0],
        "modules": seq(role_modules[1])
                   # group by (module_id, module_name)
                   .map(lambda e: ( (e.pop('module_id'), e.pop('module_name')), e) ).group_by_key()

                   # start building module/permissions dict
                   .map(lambda module_permissions: {
                       "module_id": module_permissions[0][0],
                       "module_name": module_permissions[0][1],
                       "permissions": module_permissions[1]
                   })
                   # sort by module_id, convert Seq obj to regular list
                   .sorted(key=lambda m:m['module_id']).to_list()
    })
    # sort by role_id, convert Seq obj to regular list
    .sorted(key=lambda r:r['role_id']).to_list()
)

pprint(output)
</code></pre>

<p><strong>RESULT</strong></p>

<pre><code>[{'modules': [{'module_id': 1,
               'module_name': 'ModuleOne',
               'permissions': [{'can_create': True,
                                'can_delete': True,
                                'can_read': True,
                                'can_update': True,
                                'end_point_id': 1,
                                'end_point_name': 'entity'},
                               {'can_create': True,
                                'can_delete': True,
                                'can_read': True,
                                'can_update': True,
                                'end_point_id': 2,
                                'end_point_name': 'management-type'},
                               {'can_create': True,
                                'can_delete': False,
                                'can_read': True,
                                'can_update': True,
                                'end_point_id': 3,
                                'end_point_name': 'ownership-type'}]},
              {'module_id': 2,
               'module_name': 'ModuleTwo',
               'permissions': [{'can_create': True,
                                'can_delete': True,
                                'can_read': True,
                                'can_update': True,
                                'end_point_id': 4,
                                'end_point_name': 'financial-outlay'},
                               {'can_create': True,
                                'can_delete': True,
                                'can_read': True,
                                'can_update': True,
                                'end_point_id': 5,
                                'end_point_name': 'exposure'}]}],
  'role_id': 1}]
</code></pre>

python list python-3.x dictionary