Sort and group a list of dictionaries
<p>How can I sort and group this list of dictionaries into a nested dictionary which I want to return via an API as JSON.</p>

<p>Source Data (list of permissions):</p>

<pre><code>[{
    'can_create': True,
    'can_read': True,
    'module_name': 'ModuleOne',
    'module_id': 1,
    'role_id': 1,
    'end_point_id': 1,
    'can_update': True,
    'end_point_name': 'entity',
    'can_delete': True,
}, {
    'can_create': True,
    'can_read': True,
    'module_name': 'ModuleTwo',
    'module_id': 2,
    'role_id': 1,
    'end_point_id': 4,
    'can_update': True,
    'end_point_name': 'financial-outlay',
    'can_delete': True,
},{
    'can_create': True,
    'can_read': True,
    'module_name': 'ModuleOne',
    'module_id': 1,
    'role_id': 1,
    'end_point_id': 2,
    'can_update': True,
    'end_point_name': 'management-type',
    'can_delete': True,
}, {
    'can_create': True,
    'can_read': True,
    'module_name': 'ModuleOne',
    'module_id': 1,
    'role_id': 1,
    'end_point_id': 3,
    'can_update': True,
    'end_point_name': 'ownership-type',
    'can_delete': False,
}, {
    'can_create': True,
    'can_read': True,
    'module_name': 'ModuleTwo',
    'module_id': 2,
    'role_id': 1,
    'end_point_id': 5,
    'can_update': True,
    'end_point_name': 'exposure',
    'can_delete': True,
}]
</code></pre>

<p>I want to transform that into a nested dicitonary object for return with an API as JSON. Here's the expected output:</p>

<pre><code>{
    "role_id": 1,
    "modules": [{
            "module_id": 1,
            "module_name": "ModuleOne",
            "permissions": [{
                "end_point_id": 1,
                "end_point_name": "entity",
                "can_create": False,
                "can_read": True,
                "can_write": True,
                "can_delete": True
            }, {
                "end_point_id": 2,
                "end_point_name": "management-type",
                "can_create": False,
                "can_read": True,
                "can_write": True,
                "can_delete": True
            }, {
                "end_point_id": 3,
                "end_point_name": "ownership-type",
                "can_create": False,
                "can_read": True,
                "can_write": True,
                "can_delete": True
            }, ]
        }, {
            "module_id": 2,
            "module_name": "ModuleTwo",
            "permissions": [{
                "end_point_id": 4,
                "end_point_name": "financial-outlay",
                "can_create": False,
                "can_read": True,
                "can_write": True,
                "can_delete": True
            }, {
                "end_point_id": 5,
                "end_point_name": "exposure",
                "can_create": False,
                "can_read": True,
                "can_write": True,
                "can_delete": True
            }, ]
        },

    ]
}
</code></pre>

<p>It looked trivial until I spent more time that I would expect trying to bend my mind around it. I've attempted so many options with non of them working. Here's the last attempt.</p>

<pre><code># Get user role
user_roles = get_user_roles()  # List of roles e.g. [{'role_id':1, role_name: 'role_one'}, {'role_id':2, role_name: 'role_two'}]

for role in user_roles:
    role_id = role['role_id']
    role_name = role['role_name']

    # Fetch Role Permissions
    role_permissions = get_role_permissions(role_id)  # List of permissions as seen above
    sorted_role_permissions = sorted(role_permissions, key=itemgetter('module_id'))  # sort dictionaries in list by 'module_id'

    modules_list = []
    permissions_list = []
    previous_module_id = 0
    is_first_loop = True
    for role_permission in sorted_role_permissions:

        module_id = role_permission['module_id']
        module_name = role_permission['module_name']
        end_point_id = role_permission['end_point_id']
        end_point_name = role_permission['end_point_name']

        if is_first_loop:
            print(0)
            is_first_loop = False
            previous_module_id = module_id
            print('end_point_name 0 {}'.format(end_point_name))
            permissions = {'end_point_id': end_point_id, 'end_point_name': end_point_name,
                           'can_create': role_permission['can_create'],
                           'can_read': role_permission['can_read'],
                           'can_update': role_permission['can_update'],
                           'can_delete': role_permission['can_delete']
                           }
            permissions_list.append(permissions)
            if len(sorted_role_permissions) == 1:
                # If there is only one permission in the role, end the loop
                modules_dict = {'module_id': module_id, 'module_name': module_name,
                                'permissions': permissions_list}
                modules_list.append(modules_dict)
                break

        else:

            if module_id == previous_module_id:
                # As long as the current module_id and the previous_module_id are the same, add to the same list

                print(1)
                permissions = {'end_point_id': end_point_id, 'end_point_name': end_point_name,
                               'can_create': role_permission['can_create'],
                               'can_read': role_permission['can_read'],
                               'can_update': role_permission['can_update'],
                               'can_delete': role_permission['can_delete']
                               }
                permissions_list.append(permissions)

            else:

                print(2)
                modules_dict = {'module_id': module_id, 'module_name': module_name,
                                'permissions': permissions_list}
                modules_list.append(modules_dict)
                permissions_list = []
                permissions = {'end_point_id': end_point_id, 'end_point_name': end_point_name,
                               'can_create': role_permission['can_create'],
                               'can_read': role_permission['can_read'],
                               'can_update': role_permission['can_update'],
                               'can_delete': role_permission['can_delete']
                               }
                permissions_list.append(permissions)
                previous_module_id = module_id

    if modules_list:
        roles.append({'role_id': role_id, 'role_name': role_name, 'modules': modules_list})
</code></pre>

<p>TADA!</p>

<pre><code>from itertools import groupby

def group_by_remove(permissions, id_key, groups_key, name_key=None):
    """
    @type permissions: C{list} of C{dict} of C{str} to C{object}
    @param id_key: A string that represents the name of the id key, like "role_id" or "module_id"
    @param groups_key: A string that represents the name of the key of the groups like "modules" or "permissions"
    @param name_key: A string that represents the name of the key of names like "module_name" (can also be None for no names' key) 
    """
    result = []
    permissions_key = lambda permission: permission[id_key]
    # Must sort for groupby to work properly
    sorted_permissions = sorted(permissions, key=permissions_key)
    for key, groups in groupby(sorted_permissions, permissions_key):
        key_result = {}
        groups = list(groups)
        key_result[id_key] = key
        if name_key is not None:
            key_result[name_key] = groups[0][name_key]
        key_result[groups_key] = [{k: v for k, v in group.iteritems() if k != id_key and (name_key is None or k != name_key)} for group in groups]
        result.append(key_result)
    return result

def change_format(initial):
    """
    @type initial: C{list}
    @rtype: C{dict} of C{str} to C{list} of C{dict} of C{str} to C{object}
    """
    roles_group = group_by_remove(initial, "role_id", "modules")[0]
    roles_group["modules"] = group_by_remove(roles_group["modules"], "module_id", "permissions", "module_name")
    return roles_group

change_format(role_permissions)
</code></pre>

<p>Enjoy :)</p>

python list python-3.x dictionary