AKA spiderplots AKA Dance Dance Revolution difficulty selector

def radarplot(categories, data, colors=None, title='', fill=True, facealpha=0.1):
    """Radarplot

    Parameters
    ----------
    categories: list of str
    data: dict
        key: str (name)
        value: np.array 
            - data points for each category
            - Can be 1D or 2D
            - rows = observations, columns=categories
    colors: list of matplotlib color
        - same length as data keys
    title: str
    fill: bool
    facealpha: float
    """
    
    theta_inc = (np.pi*2)/len(categories) # Angle between assays
    # theta = np.arange(0, np.pi*2, theta_inc)
    
    fig, ax = plt.subplots(figsize=(4, 4), subplot_kw={'projection': 'polar'}, dpi=100)
    # POLAR COORDINATES
    # X = theta
    # Y = r (radius)

    if not colors:
        colors = {sample: mpl.cm.tab10.colors[i] for i, sample in enumerate(data.keys())}
    if type(colors) == list:
        colors = {sample: color for sample, color in zip(data.keys(), colors)}
    
    for sample, values in data.items():
        r = []
        theta = []

        # Cast to numpy array
        if type(values) == list:
            values = np.array(values)

        # Cast to 2D array if 1D
        if len(values.shape) == 1:
            values = np.array([values])

        # For each set of observations
        for m in range(values.shape[0]):
            # For each category
            for k in range(values.shape[1]):
                r.append(values[m, k])
                theta.append(k * theta_inc)

        c = (colors[sample])
        # Reduce alpha of facecolor
        fc = tuple([x for x in c] + [facealpha])
        ec = tuple([x for x in c] + [1.0])
            
        if not fill:
            fc = 'none'
            
        ax.add_patch(mpl.patches.Polygon(
            np.vstack([theta, r]).T,
            facecolor=fc, edgecolor=ec, linewidth=0.5
        ))
        
        ax.scatter(
            theta, r, 
            facecolor='none', edgecolor=ec, s=20, 
            zorder=1000 # Plot over polygon patches
        )
        
    ax.set_xticks(theta)
    ax.set_xticklabels(categories)
    # Place labels on the outside of the plot
    # Check for left/right side of the circle
    for tick in ax.get_xticklabels():
        if tick._x > (np.pi/2) and tick._x < (3 * (np.pi/2)):
            tick.set_ha('right')
        else:
            tick.set_ha('left')
    
    # CUSTOM TICKS
    #rticks = np.arange(0, 1.1, 0.2)
    #ax.set_rlim([0, 1])
    #ax.set_rticks(rticks)
    # rgridlines, rgridlabels = ax.set_rgrids(rticks)
    # for line in rgridlines:
    #     line.set_linewidth(0.5)
    
    ax.set_title(title)
    
    legend_patches = [
        mpl.lines.Line2D([0], [0], color=colors[sample], marker='o', markerfacecolor='none', lw=1.)
        for sample in data.keys()
    ]
    ax.legend(legend_patches, list(data.keys()), 
              bbox_to_anchor=(1, 1), loc='upper left',
              frameon=False, facecolor='none')
    # plt.show()
    
    return fig, ax
    
    
fig, ax = radarplot(
    ['A', 'B', 'C', 'D', 'E'], 
    {
        'Control': norm.rvs(size=5, random_state=1),
        'Treatment': norm.rvs(size=5, random_state=2)
    }
)
plt.show()

fig, ax = radarplot(
    ['A', 'B', 'C', 'D', 'E'], 
    {
        'Control': np.reshape(norm.rvs(size=5*5, random_state=1), (5, 5)),
        'Treatment': norm.rvs(size=5, random_state=2)
    },
    
)

rlabel_pos = -1 * np.pi * (1/4)
rlabel='Fold Change'

# Radius Axis Label
if rlabel:
    ax.set_rlabel_position(np.rad2deg(rlabel_pos))
    ax.text(rlabel_pos + 0.1, 2., rlabel, 
            ha='left', va='top', rotation=np.rad2deg(rlabel_pos))
plt.show()

Basic premise is that we plot multiple histograms in their own subplot axes, and then use hspace to overlap the plotting axes vertically

import matplotlib as mpl
import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np

from scipy.stats import norm

_new_black = '#373737'
sns.set_theme(style='ticks', font_scale=0.75, rc={
    'font.family': 'sans-serif',
    'font.sans-serif': ['Arial', 'DejaVu Sans'],
    'svg.fonttype': 'none',
    'text.usetex': False,
    'pdf.fonttype': 42,
    'ps.fonttype': 42,
    'font.size': 9,
    'axes.labelsize': 9,
    'axes.titlesize': 9,
    'axes.labelpad': 2,
    'axes.linewidth': 0.5,
    'axes.titlepad': 4,
    'lines.linewidth': 0.5,
    'legend.fontsize': 9,
    'legend.title_fontsize': 9,
    'xtick.labelsize': 9,
    'ytick.labelsize': 9,
    'xtick.major.size': 2,
    'xtick.major.pad': 2,
    'xtick.major.width': 0.5,
    'ytick.major.size': 2,
    'ytick.major.pad': 2,
    'ytick.major.width': 0.5,
    'xtick.minor.size': 2,
    'xtick.minor.pad': 2,
    'xtick.minor.width': 0.5,
    'ytick.minor.size': 2,
    'ytick.minor.pad': 2,
    'ytick.minor.width': 0.5,

    # Avoid black unless necessary
    'text.color': _new_black,
    'patch.edgecolor': _new_black,
    'patch.force_edgecolor': False, # Seaborn turns on edgecolors for histograms by default and I don't like it
    'hatch.color': _new_black,
    'axes.edgecolor': _new_black,
    # 'axes.titlecolor': _new_black # should fallback to text.color
    'axes.labelcolor': _new_black,
    'xtick.color': _new_black,
    'ytick.color': _new_black

    # Default colormap - personal preference
    # 'image.cmap': 'inferno'
})

# Different vector sizes are OK - we're normalizing histograms by density
x1 = norm.rvs(loc=2, scale=1, size=1000, random_state=1)
x2 = norm.rvs(loc=3, scale=1, size=1500, random_state=2)
x3 = norm.rvs(loc=4, scale=2, size=2000, random_state=3)

# Same bins across all distributions for valid comparisons
bins = np.linspace(-3, 11, 100)

x = {
    'X1': x1,
    'X2': x2,
    'X3': x3
}

colors = mpl.cm.tab10.colors

fig = plt.figure(figsize=(4, 1.5), dpi=150)
gs = fig.add_gridspec(3, 1, hspace=-0.5, left=0.2)

for i, (label, vals) in enumerate(x.items()):
    ax = fig.add_subplot(gs[i, 0])

    ax.hist(vals, bins=bins, density=True,
        color=colors[i], edgecolor='none', alpha=0.7)
    ax.set_ylim([0, 0.5]) # Fixed ylimit for density comparisons

    # Transparent background
    ax.patch.set_alpha(0)

    # Turn off all spines except the bottom one
    for spine in ax.spines.keys():
        ax.spines[spine].set_visible(False)
        
    ax.spines['bottom'].set_visible(True)
    ax.spines['bottom'].set_linewidth(0.5)
    ax.spines['bottom'].set_color('#AAA')

    # Only show x ticks and label for bottom plot
    if i == len(x) - 1:
        ax.spines['bottom'].set_color('#444')
        ax.set_xlabel('Score')
    else:
        ax.set_xticks([])
        ax.set_xticklabels([])
    
    ax.set_yticks([])

    # Label
    ax.text(-0.01, 0, label, color=colors[i],
        transform=ax.transAxes, ha='right', va='bottom')
    
plt.show()

Here’s a similar version, but with additional graphics to display missing data (NaNs) while still appropriately scaling the histograms

# Different vector sizes are OK - we're normalizing histograms by density
x1 = norm.rvs(loc=2, scale=1, size=1000, random_state=1)
x2 = norm.rvs(loc=3, scale=1, size=1500, random_state=2)
x3 = norm.rvs(loc=4, scale=2, size=2000, random_state=3)

# Knockout half of the points
x1[:700] = np.nan

# Same bins across all distributions for valid comparisons
bins = np.linspace(-3, 11, 100)

x = {
    'X1': x1,
    'X2': x2,
    'X3': x3
}

colors = mpl.cm.tab10.colors

fig = plt.figure(figsize=(4, 1.5), dpi=150)
gs = fig.add_gridspec(3, 1, hspace=-0.5, left=0.2)

for i, (label, vals) in enumerate(x.items()):
    ax = fig.add_subplot(gs[i, 0])

    # Set NaNs to some really low value and extend the bin range to count it,
    # but use xlim to only keep our plot within the original bin range
    nan_placeholder_val = -10
    n_missing = np.sum(np.isnan(vals))
    vals[np.isnan(vals)] = nan_placeholder_val
    xlim = [bins[0], bins[-1]]
    new_bins = np.append(2 * nan_placeholder_val, bins)

    ax.hist(vals, bins=new_bins, density=True,
        color=colors[i], edgecolor='none', alpha=0.7)
    ax.set_xlim(xlim)
    ax.set_ylim([0, 0.5])
    
    # Transparent background
    ax.patch.set_alpha(0)

    # Turn off all spines except the bottom one
    for spine in ax.spines.keys():
        ax.spines[spine].set_visible(False)
        
    ax.spines['bottom'].set_visible(True)
    ax.spines['bottom'].set_linewidth(0.5)
    ax.spines['bottom'].set_color('#AAA')

    # Only show x ticks and label for bottom plot
    if i == len(x) - 1:
        ax.spines['bottom'].set_color('#444')
        ax.set_xlabel('Score')
    else:
        ax.set_xticks([])
        ax.set_xticklabels([])
    
    ax.set_yticks([])
        
    # % Not detected
    rect_width = 0.07
    rect_height = 0.5
    rect_padding = 0.01 # Padding from the axis
    ax.add_patch(mpl.patches.Rectangle(
        ((-1 * rect_width) - rect_padding, 0), rect_width, rect_height,
        transform=ax.transAxes, clip_on=False, facecolor=colors[i], alpha=0.5
    ))
    ax.text((-1 * (rect_width / 2))-rect_padding, 0.25, 
        '{:.1f}'.format((n_missing / len(vals)) * 100), 
        fontsize=7, transform=ax.transAxes, rotation=90, ha='center', va='center')
    if i == len(x) - 1:
        ax.text((-1 * (rect_width / 2))-rect_padding, -0.1, 
        '%ND', transform=ax.transAxes, ha='center', va='top', fontsize=7)

    # Label
    ax.text((-1 * rect_width) - rect_padding - 0.02, 0, label, color=colors[i],
        transform=ax.transAxes, ha='right', va='bottom')
    
plt.show()

Table of Contents

• Imports
• Theming
  • Colorblind testing
  • Categorical Colors
  • Continuous and Divergent Colormaps
• Figure sizing
  • Figure padding
• Gridspec
  • Nested gridspecs
• Scatterplots
  • Density scatterplots
    • Fast subsampled density kernels
    • Stored density kernels
  • Scatterplot rasterization
• Text annotation
  • Blended transforms
  • Point annotation
• Grids
• Colorbars

Imports

import pandas as pd
import numpy as np

import matplotlib as mpl
import matplotlib.pyplot as plt
import seaborn as sns

Theming

Matplotlib rcParams reference

Most of the seaborn defaults are sensible, so I just build around them with sns.set

For figures, I go with a default font size of 9 pt and a line size of 0.5 pt. Smaller fonts and thinner lines will be harder to read on print, but for super compact figures I sometimes go down to 7 pt font.

For theme colors, I follow Never Use Black: https://ianstormtaylor.com/design-tip-never-use-black/

_new_black = '#373737'
sns.set_theme(style='ticks', font_scale=0.75, rc={
    'font.family': 'sans-serif',
    'font.sans-serif': ['Arial', 'DejaVu Sans'],
    'svg.fonttype': 'none',
    'text.usetex': False,
    'pdf.fonttype': 42,
    'ps.fonttype': 42,
    'font.size': 9,
    'axes.labelsize': 9,
    'axes.titlesize': 9,
    'axes.labelpad': 2,
    'axes.linewidth': 0.5,
    'axes.titlepad': 4,
    'lines.linewidth': 0.5,
    'legend.fontsize': 9,
    'legend.title_fontsize': 9,
    'xtick.labelsize': 9,
    'ytick.labelsize': 9,
    'xtick.major.size': 2,
    'xtick.major.pad': 1,
    'xtick.major.width': 0.5,
    'ytick.major.size': 2,
    'ytick.major.pad': 1,
    'ytick.major.width': 0.5,
    'xtick.minor.size': 2,
    'xtick.minor.pad': 1,
    'xtick.minor.width': 0.5,
    'ytick.minor.size': 2,
    'ytick.minor.pad': 1,
    'ytick.minor.width': 0.5,

    # Avoid black unless necessary
    'text.color': _new_black,
    'patch.edgecolor': _new_black,
    'patch.force_edgecolor': False, # Seaborn turns on edgecolors for histograms by default and I don't like it
    'hatch.color': _new_black,
    'axes.edgecolor': _new_black,
    # 'axes.titlecolor': _new_black # should fallback to text.color
    'axes.labelcolor': _new_black,
    'xtick.color': _new_black,
    'ytick.color': _new_black

    # Default colormap - personal preference
    # 'image.cmap': 'inferno'
})

Colorblind testing

For figure development, try Color Oracle https://colororacle.org/, which simulates multiple different colorblindnesses and also grayscale. Ideally you should design your figure colors to also work in grayscale but in practice it’s quite difficult with more than 2 categories.

If you’re polishing figures in Illustrator, they also have colorblindness modes, in View -> Proof Setup -> Colorblindness

Categorical Colors

I use Paul Tol’s Colorblind-friendly colors for all categorical mappings. Full python code is here.

def tol_cset(colorset=None):
    """
    Discrete color sets for qualitative data.

    Define a namedtuple instance with the colors.
    Examples for: cset = tol_cset(<scheme>)
      - cset.red and cset[1] give the same color (in default 'bright' colorset)
      - cset._fields gives a tuple with all color names
      - list(cset) gives a list with all colors
    """
    from collections import namedtuple
    
    namelist = ('bright', 'high-contrast', 'vibrant', 'muted', 'medium-contrast', 'light')
    if colorset == None:
        return namelist
    if colorset not in namelist:
        colorset = 'bright'
        print('*** Warning: requested colorset not defined,',
              'known colorsets are {}.'.format(namelist),
              'Using {}.'.format(colorset)) 

    if colorset == 'bright':
        cset = namedtuple('Bcset',
                    'blue red green yellow cyan purple grey black')
        return cset('#4477AA', '#EE6677', '#228833', '#CCBB44', '#66CCEE',
                    '#AA3377', '#BBBBBB', '#000000')

    if colorset == 'high-contrast':
        cset = namedtuple('Hcset',
                    'blue yellow red black')
        return cset('#004488', '#DDAA33', '#BB5566', '#000000')

    if colorset == 'vibrant':
        cset = namedtuple('Vcset',
                    'orange blue cyan magenta red teal grey black')
        return cset('#EE7733', '#0077BB', '#33BBEE', '#EE3377', '#CC3311',
                    '#009988', '#BBBBBB', '#000000')

    if colorset == 'muted':
        cset = namedtuple('Mcset',
                    'rose indigo sand green cyan wine teal olive purple pale_grey black')
        return cset('#CC6677', '#332288', '#DDCC77', '#117733', '#88CCEE',
                    '#882255', '#44AA99', '#999933', '#AA4499', '#DDDDDD',
                    '#000000')

    if colorset == 'medium-contrast':
        cset = namedtuple('Mcset',
                    'light_blue dark_blue light_yellow dark_red dark_yellow light_red black')
        return cset('#6699CC', '#004488', '#EECC66', '#994455', '#997700',
                    '#EE99AA', '#000000')

    if colorset == 'light':
        cset = namedtuple('Lcset',
                    'light_blue orange light_yellow pink light_cyan mint pear olive pale_grey black')
        return cset('#77AADD', '#EE8866', '#EEDD88', '#FFAABB', '#99DDFF',
                    '#44BB99', '#BBCC33', '#AAAA00', '#DDDDDD', '#000000')

Continuous and Divergent Colormaps

Matplotlib’s base guide on colormaps is quite good: https://matplotlib.org/stable/tutorials/colors/colormaps.html

For continuous colormaps, always stick to “Perceptually Uniform Sequential Colormaps”, as these colormaps convert nicely to grayscale and therefore to colorblindness as well. For a visual demonstration, see: https://matplotlib.org/stable/tutorials/colors/colormaps.html#grayscale-conversion.

For divergent colormaps, since it’s impossible to differentiate high/low on grayscale, I usually use coolwarm over bwr, since the white tends to distract by blending into figure backgrounds (usually also white).

Just my opinion, but I would stay away from divergent colormaps with black as the middle color (you still might see this being used in some RNA-seq papers, with the green-black-pink] colormap). They strike me as super overwhelming and more difficult to interpret than the middle-white colormaps.

I use the matplotlib.cm module to reference named colormaps directly:

cmap = mpl.cm.inferno

fig, ax = plt.subplots(figsize=(6, 1))
gradient = np.linspace(0, 1, 256)
gradient = np.vstack((gradient, gradient))
ax.imshow(gradient, aspect='auto', cmap=cmap)
ax.set_axis_off()
plt.show()

You can get the reversed version of any colormap in matplotlib by appending _r to the colormap name

cmap = mpl.cm.inferno_r # Added _r to reverse colormap

fig, ax = plt.subplots(figsize=(6, 1))
gradient = np.linspace(0, 1, 256)
gradient = np.vstack((gradient, gradient))
ax.imshow(gradient, aspect='auto', cmap=cmap)
ax.set_axis_off()
plt.show()

Figure sizing

I like to create my figures in the exact size that they will appear on paper/in print, so that I can get a good idea of what elements I can fit and how the font sizes/line sizes will look.

It’s always hard to know exactly what your panel sizes for each plot will be inside of a figure, so I like to, if possible, block the figure out in Illustrator and use the rulers to estimate how big each panel will be.

If I don’t know your panel sizes ahead of time, then I follow these general rules:

• For standard letter size (8.5 in x 11 in), your figures/panels should not exceed 6 inches
• For standard letter size, your whole figure shouldn’t be taller than 8 inches. This leaves room for a figure caption on the same page (figure captions split across multiple pages are the worst).
• Size conservatively - more padding between panels is always better for readability

Matplotlib defaults to figure sizes in inches, which is convenient for converting to Illustrator/print. Define this with figsize

fig = plt.figure(figsize=(3, 2))

# Also works in subplots() command
fig, ax = plt.subplots(figsize=(3, 2))
from scipy.stats import norm
ax.hist(norm.rvs(size=100))
plt.show()

<Figure size 216x144 with 0 Axes>

For smaller panels (i.e., 2 x 1 inches), the output in Jupyter can be too small to view when developing the plot. You can use the dpi setting to increase the size of the image output in Jupyter

from scipy.stats import norm
x = norm.rvs(size=100)

fig, ax = plt.subplots(figsize=(3, 2), dpi=75)
ax.hist(x)
plt.show()

fig, ax = plt.subplots(figsize=(3, 2), dpi=150)
ax.hist(x)
plt.show()

Figure padding

I frequently add or remove padding around figures to account for removed axis labels, or maybe large axis labels that jut out a bit more.

You can adjust these via fig.add_gridspec() or plt.subplots_adjust() with the parameters left, right, top, and bottom. Units are in fractions of the figure [0, 1]

fig, ax = plt.subplots()
plt.subplots_adjust(left=0.2, right=0.7, bottom=0.5, top=0.9)
ax.text(-0.1, 0.5, 'A really long y axis label', transform=ax.transAxes, ha='right', va='center')
plt.show()

fig = plt.figure()
gs = fig.add_gridspec(1, 1, left=0.2, right=0.7, bottom=0.5, top=0.9)
ax = fig.add_subplot(gs[0, 0])
ax.text(-0.1, 0.5, 'A really long y axis label', transform=ax.transAxes, ha='right', va='center')
plt.show()

By default jupyter cuts off whitespace around the plot and extends the plot to cover all grapical elements - so you won’t be able to visualize your padding results in the jupyter cell output.

You can solve this by first saving the plot to an image (in a file or in a buffer), and then displaying the raw image

import io
buf = io.BytesIO()

fig = plt.figure()
gs = fig.add_gridspec(1, 1, left=0.2, right=0.7, bottom=0.5, top=0.9)
ax = fig.add_subplot(gs[0, 0])
ax.text(-0.1, 0.5, 'A really long y axis label', transform=ax.transAxes, ha='right', va='center')
# Save to buffer
buf = io.BytesIO()
fig.savefig(buf, format='png')
plt.close() # Suppress output of raw figure object

# Read image from buffer and display
from IPython.display import Image
buf.seek(0)
im = Image(data=buf.read())
buf.close()
im

Gridspec

It’s a few more lines to lay out your plots with gridspec instead of subplots but it’s totally worth it for complex layouts. Plus it also saves you from a subplots_adjust call later on.

See: https://matplotlib.org/stable/gallery/subplots_axes_and_figures/gridspec_multicolumn.html

fig = plt.figure(figsize=(4, 4))
gs = fig.add_gridspec(
    3, 3, # rows x cols
    # Cell relative width/heights
    width_ratios=[1, 1.5, 1], # Units are relative to each other
    height_ratios=[150, 100, 100],
    # Padding between cells
    wspace=0.0, # Units are in fractions of the row above/column left? I think?
    hspace=0.7
)

ax1 = fig.add_subplot(gs[0, 0])
ax2 = fig.add_subplot(gs[0, 1:3])
ax3 = fig.add_subplot(gs[1:3, 0:2])
ax4 = fig.add_subplot(gs[1, 2])
ax5 = fig.add_subplot(gs[2, 2])

plt.show()

For greating a grid of subplots, similar to subplots():

fig = plt.figure(figsize=(4, 4))

nrow = 3
ncol = 3
gs = fig.add_gridspec(nrow, ncol)

for i in range(nrow * ncol):
    row = i // ncol
    col = i % ncol
    ax = fig.add_subplot(gs[row, col])

plt.show()

Nested Gridspecs

See: https://matplotlib.org/stable/gallery/subplots_axes_and_figures/gridspec_nested.html

# From: https://matplotlib.org/stable/gallery/subplots_axes_and_figures/gridspec_nested.html

def format_axes(fig):
    for i, ax in enumerate(fig.axes):
        ax.text(0.5, 0.5, "ax%d" % (i+1), va="center", ha="center")
        ax.tick_params(labelbottom=False, labelleft=False)


# gridspec inside gridspec
fig = plt.figure()
gs0 = mpl.gridspec.GridSpec(1, 2, figure=fig)
gs00 = mpl.gridspec.GridSpecFromSubplotSpec(3, 3, subplot_spec=gs0[0])

ax1 = fig.add_subplot(gs00[:-1, :])
ax2 = fig.add_subplot(gs00[-1, :-1])
ax3 = fig.add_subplot(gs00[-1, -1])

# the following syntax does the same as the GridSpecFromSubplotSpec call above:
gs01 = gs0[1].subgridspec(3, 3)

ax4 = fig.add_subplot(gs01[:, :-1])
ax5 = fig.add_subplot(gs01[:-1, -1])
ax6 = fig.add_subplot(gs01[-1, -1])

plt.suptitle("GridSpec Inside GridSpec")
format_axes(fig)

plt.show()

Scatterplots

from scipy.stats import norm

x = norm.rvs(size=100)
y = x + norm.rvs(0, 0.3, size=100)

fig = plt.figure(figsize=(2, 2), dpi=150)
gs = fig.add_gridspec(1, 1)
ax = fig.add_subplot(gs[0, 0])
ax.scatter(x, y)
plt.show()

By default, matplotlib/seaborn also plots an edge around each scatterplot marker that limits the minimum size of points. If you specify edgecolor='none', then you can get correct point sizes

fig = plt.figure(figsize=(6, 2), dpi=150)
gs = fig.add_gridspec(1, 3)
ax = fig.add_subplot(gs[0, 0])
ax.scatter(x, y, s=1)
ax.set_title('Point size 1.0')

ax = fig.add_subplot(gs[0, 1])
ax.scatter(x, y, s=0.2)
ax.set_title('Point size 0.2')

ax = fig.add_subplot(gs[0, 2])
ax.scatter(x, y, s=0.2, edgecolor='none')
ax.set_title('Point size 0.2, no edge color')
plt.show()

Density scatterplots

For huge datasets of thousands of points, scatterplots will result in points plotting over each other and hiding the underlying distribution of data. For middling datasets you can get away with this by adjusting the alpha of each point, but then it might get difficult to see individual points. It’s better to apply a density kernel as a color on top of each point

from scipy.stats import norm

x = norm.rvs(size=10_000)
y = x + norm.rvs(0, 0.5, size=10_000)

fig = plt.figure(figsize=(8, 2), dpi=150)
gs = fig.add_gridspec(1, 4)

ax = fig.add_subplot(gs[0, 0])
ax.scatter(x, y)
ax.set_title('Base color')

ax = fig.add_subplot(gs[0, 1])
ax.scatter(x, y, s=1, edgecolor='none')
ax.set_title('Smaller points')

ax = fig.add_subplot(gs[0, 2])
ax.scatter(x, y, s=1, edgecolor='none', alpha=0.2)
ax.set_title('Smaller points + alpha')

from scipy.stats import gaussian_kde
kernel = gaussian_kde(np.vstack([x, y]))
c = kernel(np.vstack([x, y]))

ax = fig.add_subplot(gs[0, 3])
ax.scatter(x, y, s=1, edgecolor='none', c=c, cmap=mpl.cm.viridis)
ax.set_title('Density Kernel')

fig.suptitle('N=10,000 points', y=1.05)
plt.show()

Fast subsampled density kernels

For huge datasets, the density kernel takes a really long time to calculate. When developing plots for these huge datasets, I subsample the kernel for faster run times. The result is usually indistinguishable (although for final publication I re-generate with the fully sampled density kernel)

from scipy.stats import norm

x = pd.Series(norm.rvs(size=10_000)) # Use pandas Series for the sample function
y = x + norm.rvs(0, 0.5, size=10_000)

fig = plt.figure(figsize=(4, 2), dpi=150)
gs = fig.add_gridspec(1, 2)

from scipy.stats import gaussian_kde
kernel = gaussian_kde(np.vstack([x, y]))
c = kernel(np.vstack([x, y]))

ax = fig.add_subplot(gs[0, 0])
ax.scatter(x, y, s=1.0, edgecolor='none', c=c, cmap=mpl.cm.viridis)
ax.set_title('Full Sample (n=10,000)')

# Kernel subsampling
# Maintain random state between x and y, otherwise we'll pull different samples
# from each vector
kernel = gaussian_kde(np.vstack([
    x.sample(n=1000, random_state=1), 
    y.sample(n=1000, random_state=1)
]))
c = kernel(np.vstack([x, y]))

ax = fig.add_subplot(gs[0, 1])
ax.scatter(x, y, s=1.0, edgecolor='none', c=c, cmap=mpl.cm.viridis)
ax.set_title('Subsampled (n=1000)')

plt.show()

Stored density kernels

For full density kernels, you could also store the color vectors in a list/dictionary in a prior cell, so that you can iterate on your plot without regenerating the kernel and its estimates every run

n_datasets = 4

xs = []
ys = []
for i in range(n_datasets):
    xs.append(norm.rvs(size=10_000)) # Use pandas Series for the sample function
    ys.append(xs[i] + norm.rvs(0, (i+1) / 4, size=10_000))

colorvecs = []
for i in range(n_datasets):
    x = xs[i]
    y = ys[i]
    kernel = gaussian_kde(np.vstack([x, y]))
    colorvecs.append(kernel(np.vstack([x, y])))

fig = plt.figure(figsize=(4, 4), dpi=100)
gs = fig.add_gridspec(2, 2)

for i in range(n_datasets):
    ax = fig.add_subplot(gs[i // 2, i % 2])
    x = xs[i]
    y = ys[i]
    c = colorvecs[i]
    ax.scatter(x, y, c=c, cmap=mpl.cm.viridis, s=1.0, edgecolor='none')

plt.show()

Scatterplot rasterization

If moving large scatterplot figures into another program (i.e., Illustrator), saving as vector graphics (SVG, PDF) will make every point an object and make the resulting file impossible to open/edit. Saving the entire plot as an image (JPEG/PNG) is also undesirable since we would want the line/text elements (axis labels, etc) to remain as vector graphics.

The solution is to pass rasterized=True into the scatter() call, so that only the scatterplot points are rasterized, while the rest of the plot stays vectorized.

At small figure sizes/low DPIs, the rasterization may be too apparent (too low resolution). You can solve this by bumping up the DPI of the figure

x = norm.rvs(size=10_000)
y = x + norm.rvs(0, 0.5, size=10_000)

fig = plt.figure(figsize=(4, 2), dpi=100)
gs = fig.add_gridspec(1, 2)

kernel = gaussian_kde(np.vstack([x, y]))
c = kernel(np.vstack([x, y]))

ax = fig.add_subplot(gs[0, 0])
ax.scatter(x, y, s=1, c=c, cmap=mpl.cm.viridis, edgecolor='none')
ax.set_title('Vector')

ax = fig.add_subplot(gs[0, 1])
ax.scatter(x, y, s=1, c=c, cmap=mpl.cm.viridis, rasterized=True, edgecolor='none')
ax.set_title('Rasterized')

plt.show()

Text annotation

Placing text annotations on plots can be tricky if using the same coordinate system as the data - different scales will require a lot of custom code to tweak the exact location of each text label.

Instead, use transform=ax.transAxes to place text relative to the axis box, instead of in the data coordinate system

x = norm.rvs(size=10_000, random_state=1)
y1 = x + norm.rvs(size=10_000, random_state=2)
y2 = 10 * (x + norm.rvs(size=10_000, scale=0.5, random_state=3))

fig = plt.figure(figsize=(4, 2), dpi=100)
gs = fig.add_gridspec(1, 2, wspace=0.4)

ax = fig.add_subplot(gs[0, 0])
kernel = gaussian_kde(np.vstack([x, y1]))
c = kernel(np.vstack([x, y1]))

ax.scatter(x, y1, c=c, cmap=mpl.cm.viridis, s=1.0, edgecolor='none')
ax.text(0.05, 0.95, r'$\rho$ = {:.2f}'.format(np.corrcoef(x, y1)[0, 1]), 
    transform=ax.transAxes, ha='left', va='top')

ax = fig.add_subplot(gs[0, 1])
kernel = gaussian_kde(np.vstack([x, y2]))
c = kernel(np.vstack([x, y2]))

ax.scatter(x, y2, c=c, cmap=mpl.cm.viridis, s=1.0, edgecolor='none')
ax.text(0.05, 0.95, r'$\rho$ = {:.2f}'.format(np.corrcoef(x, y2)[0, 1]), 
    transform=ax.transAxes, ha='left', va='top')


plt.show()

Blended transforms

Guide: https://matplotlib.org/stable/tutorials/advanced/transforms_tutorial.html#blended-transformations

Sometimes it’s useful to have one coordinate relative to the axis, and another relative to the data. For example, annotating a significance cutoff on a plot:

from scipy.stats import expon

x = norm.rvs(size=10_000, random_state=2)
y = np.abs(x) * expon.rvs(scale=0.5, size=10_000, random_state=2)

kernel = gaussian_kde(np.vstack([x, y]))
c = kernel(np.vstack([x, y]))

fig = plt.figure(figsize=(2, 1.5), dpi=150)
gs = fig.add_gridspec(1, 1, wspace=0.4)

ax = fig.add_subplot(gs[0, 0])
ax.scatter(x, y, s=1.0, c=c, cmap=mpl.cm.viridis, edgecolor='none')
ax.axhline(2, color='r', linewidth=0.5)
ax.text(0.03, 2.2, 'FDR < 1%', color='red', fontsize=8,
    transform=mpl.transforms.blended_transform_factory(ax.transAxes, ax.transData),
    rotation=90, ha='left', va='bottom'
)
ax.axvline(0, color='gray', linewidth=0.5, linestyle='--')
ax.set_yticks(np.arange(0, 9))
ax.set_xlabel('Log FC')
ax.set_ylabel('–Log10 FDR')

plt.show()

Point annotation

adjustText is a great package (inspired from ggrepel) to annotate individual points: https://github.com/Phlya/adjustText

• adjustText guide: https://github.com/Phlya/adjustText/wiki
• adjustText API: https://adjusttext.readthedocs.io/en/latest/
• annotate API: https://matplotlib.org/stable/api/_as_gen/matplotlib.pyplot.annotate.html
• Arrow parameters: https://matplotlib.org/stable/api/_as_gen/matplotlib.patches.FancyArrowPatch.html#matplotlib.patches.FancyArrowPatch

from scipy.stats import expon
from adjustText import adjust_text

x = norm.rvs(size=10_000, random_state=2)
y = np.abs(x) * expon.rvs(scale=0.5, size=10_000, random_state=2)

kernel = gaussian_kde(np.vstack([x, y]))
c = kernel(np.vstack([x, y]))

fig = plt.figure(figsize=(2, 1.5), dpi=150)
gs = fig.add_gridspec(1, 1, wspace=0.4)

ax = fig.add_subplot(gs[0, 0])
scatter = ax.scatter(x, y, s=1.0, c=c, cmap=mpl.cm.viridis, edgecolor='none')
ax.axhline(2, color='r', linewidth=0.5)
ax.text(0.03, 2.2, 'FDR < 1%', color='red', fontsize=8,
    transform=mpl.transforms.blended_transform_factory(ax.transAxes, ax.transData),
    rotation=90, ha='left', va='bottom'
)
ax.axvline(0, color='gray', linewidth=0.5, linestyle='--')
ax.set_yticks(np.arange(0, 9))
ax.set_xlabel('Log FC')
ax.set_ylabel('–Log10 FDR')

sig = (x > 1.8) & (y > 4.5)
texts = [
    plt.text(_x, _y, str(i), ha='center', va='center', fontsize=7) 
    for i, (_x, _y) in enumerate(zip(x[sig], y[sig]))
]
adjust_text(texts, add_objects=[scatter], expand_points=(1.5, 1.5),
arrowprops=dict(arrowstyle='-', color='#AAA', mutation_scale=3, linewidth=0.5))

# Make significant points bigger
ax.scatter(x[sig], y[sig], s=3, color='r', edgecolor='none')

plt.show()

/Users/chena/miniconda3/lib/python3.7/site-packages/matplotlib/transforms.py:659: RuntimeWarning: invalid value encountered in double_scalars
  deltaw = (sw * width - width) / 2.0
/Users/chena/miniconda3/lib/python3.7/site-packages/matplotlib/transforms.py:660: RuntimeWarning: invalid value encountered in double_scalars
  deltah = (sh * height - height) / 2.0

Grids

IMO grids are distracting and should only be used when comparing values is crucial to the message of the figure.

For example, grids in scatterplots are generally not useful. For plots like volcano plots where it’s necessary to visualize a significance cutoff (e.g., FDR < 1%), it’s more efficient to draw just the cutoff and still forgo the grid.

x = norm.rvs(size=10_000)
y = x + norm.rvs(0, 0.5, size=10_000)

fig = plt.figure(figsize=(4, 2), dpi=100)
gs = fig.add_gridspec(1, 2)

kernel = gaussian_kde(np.vstack([x, y]))
c = kernel(np.vstack([x, y]))

ax = fig.add_subplot(gs[0, 0])
ax.scatter(x, y, s=1, c=c, cmap=mpl.cm.viridis, edgecolor='none')
ax.grid(linewidth=0.5)
ax.set_title('Grid')

ax = fig.add_subplot(gs[0, 1])
ax.scatter(x, y, s=1, c=c, cmap=mpl.cm.viridis, rasterized=True, edgecolor='none')
ax.set_title('No Grid')

plt.show()

Of course the use of the grid is stylistic and also dependent on the message. Does it matter that the reader can use the grid to know that the top points are between 10e-6 and 10e-8? Or is it sufficient to just communicate that those points are way above the other hits and our significance cutoff?

from scipy.stats import expon

x = norm.rvs(size=10_000, random_state=2)
y = np.abs(x) * expon.rvs(scale=0.5, size=10_000, random_state=2)

kernel = gaussian_kde(np.vstack([x, y]))
c = kernel(np.vstack([x, y]))

fig = plt.figure(figsize=(4, 1.5), dpi=150)
gs = fig.add_gridspec(1, 2, wspace=0.4)

ax = fig.add_subplot(gs[0, 0])
ax.scatter(x, y, s=1.0, c=c, cmap=mpl.cm.viridis, edgecolor='none')
ax.set_yticks(np.arange(0, 9))
ax.grid(linewidth=0.5)
ax.set_xlabel('Log FC')
ax.set_ylabel('–Log10 FDR')
ax.set_title('Grid')

ax = fig.add_subplot(gs[0, 1])
ax.scatter(x, y, s=1.0, c=c, cmap=mpl.cm.viridis, edgecolor='none')
ax.axhline(2, color='r', linewidth=0.5)
ax.text(0.03, 2.2, 'FDR < 1%', color='red', fontsize=8,
    transform=mpl.transforms.blended_transform_factory(ax.transAxes, ax.transData),
    rotation=90, ha='left', va='bottom'
)
ax.axvline(0, color='gray', linewidth=0.5, linestyle='--')
ax.set_yticks(np.arange(0, 9))
ax.set_xlabel('Log FC')
ax.set_ylabel('–Log10 FDR')
ax.set_title('No grid')

plt.show()

For this example, it probably matters more where exactly each bar lies. We’re comparing a two values, and the scale is not as stretched out as the volcano plot so it matters more to know the exact location between our y-ticks. In this case the grid is helpful

x = np.arange(0, 5)
y1 = norm.rvs(loc=2, scale=1, size=5, random_state=1)
y2 = norm.rvs(loc=3, scale=1, size=5, random_state=2)
bar_width = 0.4

fig = plt.figure(figsize=(4, 1.5), dpi=150)
gs = fig.add_gridspec(1, 2, wspace=0.4)

ax = fig.add_subplot(gs[0, 0])
ax.bar(x - (bar_width/2) , y1, width=bar_width, label='Control')
ax.bar(x + (bar_width/2) , y2, width=bar_width, label='Treatment')
ax.set_xticks(x)
ax.set_yticks(np.arange(0, 5))
ax.set_xlabel('Conditions')
ax.set_ylabel('Fold change')
ax.set_title('Grid', pad=40)
ax.grid(axis='y', linewidth=0.5)
ax.legend(loc='lower left', bbox_to_anchor=(0,1))

ax = fig.add_subplot(gs[0, 1])
ax.bar(x - (bar_width/2) , y1, width=bar_width, label='Control')
ax.bar(x + (bar_width/2) , y2, width=bar_width, label='Treatment')
ax.set_xticks(x)
ax.set_yticks(np.arange(0, 5))
ax.set_xlabel('Conditions')
ax.set_ylabel('Log Fold Change')
ax.set_title('No grid', pad=40)
ax.legend(loc='lower left', bbox_to_anchor=(0,1))

plt.show()

Colorbars

I prefer to manually allocate space for colorbars instead of trusting matplotlib to create a sensible placement. This also gives you more direct control over the spacing and the axes (instead of having to rely on proxy functions on the created colorbar object).

Instead of tuning width_ratio on subplot() or gridspec(), I create separate gridspecs for the plot and the colorbar, which let’s me tune the spacing with the padding keyword arguments

I first test this out by creating empty axes to see where things will land:

x = norm.rvs(size=10_000)
y = x + norm.rvs(0, 0.5, size=10_000)

fig = plt.figure(figsize=(2.2, 2), dpi=150)
gs = fig.add_gridspec(1, 1, right=0.9)

# kernel = gaussian_kde(np.vstack([x, y]))
# c = kernel(np.vstack([x, y]))

ax = fig.add_subplot(gs[0, 0])
# ax.scatter(x, y, s=1, c=c, cmap=mpl.cm.viridis, edgecolor='none')

# Colorbar grid and axes
gs_cbar = fig.add_gridspec(1, 1, left=0.92, right=0.95)
ax = fig.add_subplot(gs_cbar[0, 0])

plt.show()

Then I fill in, and pass the scatterplot object to the colorbar as a mappable.

I try to avoid sideways text wherever possible, and to instead place the colorbar label on top and offset of the colorbar for easier reading

x = norm.rvs(size=10_000)
y = x + norm.rvs(0, 0.5, size=10_000)

fig = plt.figure(figsize=(2.2, 2), dpi=150)
gs = fig.add_gridspec(1, 1, right=0.9)

kernel = gaussian_kde(np.vstack([x, y]))
c = kernel(np.vstack([x, y]))

ax = fig.add_subplot(gs[0, 0])
scatter = ax.scatter(x, y, s=1, c=c, cmap=mpl.cm.viridis, edgecolor='none')

# Colorbar grid and axes
gs_cbar = fig.add_gridspec(1, 1, left=0.92, right=0.95)
ax = fig.add_subplot(gs_cbar[0, 0])
fig.colorbar(scatter, cax=ax)
ax.text(1, 1.01, 'Density', transform=ax.transAxes, ha='right', va='bottom')

plt.show()

If you have multiple panels, or multiple mappables to pass to colorbar, it’s simple to define the mapping yourself. Just remember that if you set vmin and vmax manually on the colorbar, to also enforce the same limits on your other plots as well.

x = norm.rvs(size=10_000)
y = x + norm.rvs(0, 0.5, size=10_000)

fig = plt.figure(figsize=(2.2, 2), dpi=150)
gs = fig.add_gridspec(1, 1, right=0.9)

kernel = gaussian_kde(np.vstack([x, y]))
c = kernel(np.vstack([x, y]))

ax = fig.add_subplot(gs[0, 0])
scatter = ax.scatter(x, y, s=1, c=c, vmin=0, vmax=0.3, cmap=mpl.cm.viridis, edgecolor='none')

# Colorbar grid and axes
gs_cbar = fig.add_gridspec(1, 1, left=0.92, right=0.95)
ax = fig.add_subplot(gs_cbar[0, 0])
fig.colorbar(
    mpl.cm.ScalarMappable(
        norm=mpl.colors.Normalize(vmin=0, vmax=0.3), 
        cmap=mpl.cm.viridis
    ), 
    cax=ax
)
ax.text(1, 1.01, 'Density', transform=ax.transAxes, ha='right', va='bottom')

plt.show()

Zotero is a fantastic non-profit, open source paper and reference manager with a ton of cool features. I mostly use it to organize and annotate papers, but it can be used for a lot, lot more!

While Zotero provides integrated library storage as part of the standalone application, I quickly overran the 300MB free tier, and didn’t want to worry about data quotas when I was adding papers to my library. (Note, you can still support Zotero in other ways.) Instead of storing paper files with Zotero, you can easily connect Zotero with a cloud storage provider like Google Drive, Dropbox, Box, Sync etc.

There’s a great guide on doing this with the Zotfile plugin for Zotero, and while that guide adds more functionality I found the following steps to be a simpler and still effective way of integrating Zotero with cloud storage.

1. Create an account on zotero.org

This is free. Note that if syncing libraries between devices, Zotero will collect data as part of the syncing process, but has no financial incentive to sell your data.

2. Download Zotero standalone

You can download the Zotero desktop app here: https://www.zotero.org/download/

3. Find the Zotero data directory

On the top bar, go to Zotero → Preferences, and in the “Advanced” tab, find the “Data Directory Location”. This is where your PDFs are normally stored and linked to through Zotero.

4. Symlink the data directory with cloud storage

This requires that your cloud storage provider has filesystem integration - which nearly all do. I’m using Google Drive File Stream which mounts my cloud storage at /Volumes/GoogleDrive/. A service like Dropbox could link its sync folder in /Users/<user>/Dropbox, for example. Create a folder in the synced directory for storing PDFs.

mkdir "/Volumes/GoogleDrive/My Drive/papers"

In the data directory folder, make a backup copy of your storage/ directory, and then delete the original directory. Then, make a symbolic link (symlink) to the synced directory you created earlier.

# Remove the original storage/ folder. Make a backup first!!!
rm -r /Users/chena/Zotero/storage
# Make a symlink to the folder in the synced directory
ln -s "/Volumes/GoogleDrive/My Drive/papers" "/Users/chena/Zotero/storage"

On Windows, the above syntax will be different:

# Create a "directory junction" - windows version of a symlink
mklink /J "G:\My Drive\papers" "C:\Users\chena\Zotero\storage"

5. Copy original data to the synced folder

If you already had files stored in the original storage/ directory, then move the contents of that directory back into the new storage/ directory link, which now points to the synced folder.

Now our storage/ folder points to a synced folder, backed up by our cloud storage provider. (On Mac, that little arrow under the folder indicates the folder is a symlink)

6. Enable Zotero Sync

Go to Zotero → Preferences, and in the “Sync” tab, sign into your Zotero account. Enable “Automatic Sync” if you want changes to automatically push to their server. By disabling the “File Syncing” options, we are no longer uploading PDFs to Zotero’s servers, and instead using our symlinked storage folder to carry files across devices.

That’s it! Now you can store unlimited papers (or whatever your data quota is on your cloud storage provider) across multiple devices.

Thanks to the Zotero team for creating a great reference management solution. It’s great.

The code is posted as a gist here: https://gist.github.com/atc3/f0260a99c5328a12f4b6fbbe65beec83, as a python function and a CSV file for the PET91 substitution matrix (Thanks to @ACRMGroup for digitizing it).

The conservation function I’m using is the “Valdar conservation score” described in Valdar and Thornton, 2001. It’s written from scratch and validated by checking the output on the same set of sequences when compared to the same Valdar score generated in SnapGene Viewer. Input a list of aligned sequences (I’m using Clustal Omega for multiple sequence alignment), and the function returns a list of conservation scores - one for each base in the alignment. The score is scaled from 0 to 1, where 0 is not conserved at all, and 1 is perfectly conserved.

I’ve split up the code into some general parts so it should be simple to use a different substitution matrix (BLOSUM, etc) or a different amino acid similarity function entirely.

Northeastern University recently got a brand-new re-branding, from the great designers over at Upstatement. My first attempts at a sweater-friendly logo were closely led by Northeastern’s Brand Book.

Typography

Some vanilla stuff, based on the strict guidelines in the brand book:

Lab Branding

We next decided to try to incorporate our rainbow colors from last round

For the second row here, I followed the advice of my friend Mike and saturated/darkened some of the lighter colors.

It also doesn’t look bad stacked vertically:

Ribosomes!

Our lab likes ribosomes, so we decided to try to make a graphic out of a bunch of them translating on an mRNA!

Getting our ribosomes to look less like birds on a power line…

And finally, our design on different backgrounds:

• Intro
• Background
• Building the List
  • Getting the Data
  • Generating the List
• Running the Experiment
  • Instrument Settings
• Appendix
  • Inclusion List Format

Intro

Running data-dependent acquisiton (DDA) mass spectrometry experiments in targeting mode is very powerful and desirable if you are trying to get around the bias towards more intense ions inherent in the data-dependent algorithm. For single cell proteomics methods (SCoPE-MS, mPOP) especially, where short methods and long injection times yield at most 10K MS/MS events per experiment, we may need targeting in order to analyze low-abundance pathways of interest.

Of course SRM/PRM methods are more sensitive, but they’re more limited in the number of proteins analyzed – targeted DDA is a nice middle ground to get what we want and a lot of it too.

MaxQuant.Live (link to manuscript) is a great alternative to targeted DDA in Xcalibur, but it does require some extra set-up and tuning. Targeting in Xcalibur is less powerful, but may be more convenient for some users.

Background

Conventional DDA performs a survey scan (MS1 scan) on all ions passing through our MS, and then selects the N most intense ions (“Top N”) to analyze sequentially. It only will select an ion if it is not on an exclusion list and if it passes the AGC Min threshold. The algorithm will then accumulate the ion in an ion trap until either 1) the accumulation time hits the maximum injection time (IT) or 2) the number of ions hits the AGC Max threshold. Only then will the collected ions be released into the mass analyzer.

When we put DDA into targeting mode, we give it a list of ions to look at. Instead of picking the N most intense ions after a survey scan, it will pick the N most intense ions that are also on the provided list of ions. These ions still have to pass the AGC Min threshold and the accumulation procedure is the same for non-targeted DDA.

Building the List

Getting the Data

In order to tell the instrument to target ions, we must first identify them. You can build a targeting list from previous runs, but if these previous runs were run on conventional non-targeted DDA methods then we will be passing the instrument a list of the most abundant ions – defeating the purpose of running targeted DDA in the first place.

We want to analyze and sequence as many ions as we can, especially low-abundnce peptides that would normally be ignored by Top N DDA. We also want to match LC-MS/MS conditions as best as possible to the eventual targeted run, so fractionating or running a super-long gradient is not optimal.

To achieve this, we can:

1) Run the sample of interest a few times back-to-back, but before each run add the IDs from the previous run to an exclusion list. While the first run will select only the most abundant ions, each successive run will select less and less abundant ions. Generating an exclusion list is almost the same process as generating the inclusion list, which is explained below.

2) Run a bulkier sample with instrument settings tuned to maximize IDs at the expense of quantification quality. For low-abundance TMT sets, for example, this could mean decreasing the max injection time and tuning the NCE to maximize identifications instead of to maximize the number of TMT reporter ions (normally: 33 NCE → 28 NCE)

Generating the List

All we have to provide the MS with for targeted DDA is 1) the target m/z, 2) the retention time window, and 3) the target ion’s charge. These are all fields that we can find from search results, such as from the evidence.txt output from a MaxQuant search. Reformatting your data is then as simple as a short script or copy/pasting columns in Excel.

Our lab uses MaxQuant, but since many of our experiments are TMT-tagged, and since MaxQuant does not calculate masses and m/z correctly for TMT-labelled peptides, we have to calculate the mass on our own. I wrote a script to do this and a few other useful things, such as simple retention time alignment. You can find it here: https://gist.github.com/atc3/84c35bc900f746fa21a558e278efc67c

Running the Experiment

In XCalibur, open the Methods Editor and go to the tab for your mass spec. At the top of the MS method, click on “Global Lists”:

From here you can specify inclusion and exclusion lists, among others. Click on “Inclusion List” to edit the method’s inclusion list.

A dialog box should pop up with an excel-like editable table.

If you don’t want to enter in values manually, you can import a .csv (such as the one generated with my script linked above)

Instrument Settings

To turn inclusion on, simply set the “inclusion” parameter to on in the method properties sidebar.

One other important behavior is what the instrument should do when it can’t find anything on the inclusion list. Should it do nothing? Or go for other peaks anyways? To expose this option first switch into advanced user mode, in the method properties sidebar:

Then at the bottom you can choose your desired behavior.

Appendix

Thanks to Ed Emmott for help & knowledge.

Inclusion List Format

Details taken from a QE-Plus software manual and the XCalibur help pages.

Mass [m/z]: Enter the mass-to-charge-ratio of the inclusion mass into the field (with a maximum of five decimals). NOTICE: If you use the Mass Calculator to edit the Formula field, this field is automatically updated when you exit the editor.

Formula [M]: Use this field to describe the structure of the active compound. In addition to chemical formulas, you can enter amino acid sequences (one letter code) and peptide sequences (if they are structural terminated by H₂O). The different ways of definition are called formula types. You can either enter the formula directly or use the Mass Calculator. To display this editor, click the down arrow in this field.

If more than one formula type for your input is possible, the Method Editor chooses the type with the highest priority (chemical formula > peptides > amino acids). Peptide sequences are marked by the prefix “p:” (for example, p: MRFA with m/z = 524.26496), amino acids by the prefix “aa:” (for example, aa: MRFA with m/z = 506.25440). Enter a prefix directly with the formula to ensure that the right type is applied.

Pay attention to the following restrictions:

• When you enter a valid formula, the Method Editor will set the default values for charge state (1) and Species (+H/-H). Then it will display an automatically calculated value in the Mass [m/z] field.

• When you enter a formula that the Method Editor cannot interpret, the Mass [m/z] field will stay empty. When you save the method, the Method Editor will warn you that the method is not in the shape to be executed (due to the missing m/z value). When you reopen the method, the Method Editor will display exactly the formula that was entered.

• When you enter a formula, The Method Editor requires a value for the charge state to calculate an m/z value. You can, however, save the method without specifying a charge state for the entry. When you save the method, the Method Editor will warn you that the method is not in the shape to be executed (due to the missing charge state). When you reopen the method, the Method Editor will display exactly the formula that was entered.

• Entries of compounds with calculated m/z values outside the measurement specifications of the mass spectrometer (m/z=50–6000) can be saved in an instrument method. The instrument software, however, will ignore these entries when it executes the method.

• When you change the m/z value that was calculated for the given set of formula, species and charge state, the Method Editor will clear the formula and species definition. The charge state will be preserved.

• When you change the polarity for a set of calculated m/z, formula, species and charge state, the Method Editor will clear the Mass [m/z] field if the species definition is not valid for this polarity. You then need to change the species definition.

Species: Use this text field to define adducts or modifications of the active compound that are expected to be formed. The default value is +H for positive polarity and –H for negative polarity. The Method Editor will then recalculate the value in the Mass [m/z] field, if possible.

The Species field can be used in two ways:

• Click the down arrow to display a list of predefined adducts for each polarity:

• +H, +Na, +K, +NH4 for positive polarity
• -H, +Cl, +OH, +HCOO for negative polarity
• an empty entry to express adducts (for example, radical cations) for both polarities.

The corresponding set will be displayed depending on the selected polarity.

Selecting an adduct (A) will result in the strict behavior of applying one unit A to the compound and using “+H” or “-H” adduct depending on the charge state and active polarity.

“+K” = [M + K⁺ + (z-1)H⁺]^z+; for example, MRFA, +K, CS=2, positive pol. = [M + K⁺+ H+]²⁺

Enter the modifications of the compound by using squared brackets and at least M as representation of the basic compound (for example, [M + Na + K] or [2M + Na]). The predefined adducts can be entered, too. This definition is used without additional auto dependencies, like adding protons.

NOTICE: If you use the Mass Calculator to edit the Formula field, the Species field is automatically filled when you exit the editor.

CS [z]: Use this spin box to enter the charge state of the ion to be fragmented. The valid range is 1–25. The required collision energy for fragmenting an ion depends on its charge state. The higher the charge state, the lower the required collision energy. If this field is left blank, any charge state will be accepted, even unassigned.

NOTICE If you use the Mass Calculator to edit the Formula field, this field is automatically updated when you exit the editor.

Polarity: Use this list box to toggle between positive ion and negative ion polarity. During a scan, the mass spectrometer uses only the inclusion masses with a polarity that matches the active ion mode.

NOTICE If you use the Mass Calculator to edit the Formula field, this field is automatically updated when you exit the editor

Start [min]: Enter the start of the time window (in minutes) that corresponds to the mass in this row. Alternatively, you can enter a time in seconds (by appending the letter s) or in hours (by appending the letter h). The software automatically converts your input into minutes.

End [min]: Enter the end of the time window in minutes. Alternatively, you can enter a time in seconds (by appending the letter s) or in hours (by appending the letter h). The software automatically converts your input into minutes.

(N)CE: Use the spin box to enter a potential gradient in the HCD cell used to fragment ions. The default collision energy is 35 eV. In the list box, specify whether absolute collision energy CE [eV] is used or normalized collision energy NCE is used. The valid range is 10 to 200 eV for CE or 10 to 200 for NCE. In the list, the software adds the prefix “ce:” to collision energy values (for example, ce:35).

A high collision energy value results in more energy deposition (which generally leads to more fragmentation). A low collision energy value results in less energy deposition (which generally leads to less fragmentation).

MSX ID: Enter the number of the multiplexed scan event in which the inclusion mass was analyzed.

Comment: Enter a comment for the inclusion mass into the field (for example, the compound name). This field is optional.

First, we have to enable the extension by passing a few flags to the renderDataTable call. Since Shiny doesn’t have a JS –> R hook built-in for the row-reorder event, we have to make it manually, with some custom javascript code.

## server.R

# example data
get_table <- reactive({
    data.frame(
        A=LETTERS[1:10]
        B=rnorm(10)
    )
})

output$table <- DT::renderDataTable({
    get_table()
}, extensions='RowReorder', options=list(rowReorder=T), callback=JS(
    "// pass on data to R
    table.on('row-reorder', function(e, details, changes) {
        Shiny.onInputChange('table_row_reorder', JSON.stringify(details));
    });"
))

## ui.R
DT::dataTableOutput('table')

Now, when we reorder the rows in our table, we will catch this event in our javascript and pass back the details of the reorder event via. the shiny event table_row_reorder. To change the underlying order of the table from this event, we first need to listen to the table_row_reorder event in the server.

# store the order of the table as a reactive value that we
# can set and also listen for changes
table_order <- reactiveVal(value=seq(1,10))

# observe row reordering event - sent from javascript function
observeEvent(input$table_row_reorder, {
    info <- input$table_row_reorder
    # error checking
    if(is.null(info) | class(info) != 'character') { return() }

    # I'm using the "yaml" package to read JSON, but you can use "jsonlite" 
    # if you want. just be aware that jsonlite also uses the "validate" 
    # function which conflicts with the same "validate" function in shiny.
    info <- read_yaml(text=info)
    # info will be empty if a reorder event fired but no row orders changed
    if(length(info) == 0) { return() }

    # load our order vectors
    .order <- table_order()
    .new_order <- .order

    # for each updated row in the info object, update the order vector
    for(i in 1:length(info)) {
        j <- info[[i]]
        .new_order[(j$newPosition + 1)] <- .order[(j$oldPosition + 1)]
    }

    # update our order vector's reactive value
    table_order(.new_order)
})

Here’s a slightly more complex version in action, inside DO-MS:

Bugs, Glitches

• The left row-number column doesn’t change when reordering, but you could create your own “original order” column to let the user keep track of the original order
• There are some visual glitches/flashes when the table is updating. I understand that this can be fixed by modifying the underlying datatables instance directly with a dataTableProxy() object (as described here), but I’ve had some issues with it and can’t get it to work

(MaxQuant Version 1.6.5.0 - Home Page)

The popular proteomics search engine MaxQuant has recently gone Linux, after getting their code base to work with the Mono/C# runtime.

The official documentation on getting MaxQuant to run on Linux is sparse, so this repo is dedicated to passing on small tips and helper scripts that help people get started.

All code is hosted in this GitHub repo

Table of Contents

• Requirements
• Running MaxQuant
• Partial Processing
• Helper Scripts

Requirements

• A decent computer
• Mono Framework >= 5.4.1
  • >> Install Mono for Linux
  • Check version: mono --version

Note, If searching Bruker timsTOF data, you need gcc version 8.3+. Check the version with gcc --version. You can install gcc8 with these commands:

# For CentOS 7:

yum install centos-release-scl
yum install devtoolset-8
scl enable devtoolset-8 -- bash

# Enable the tools (put this into .bash_profile)
source /opt/rh/devtoolset-8/enable

(Thanks to @cscaife for identifying this step, and https://stackoverflow.com/a/55876012 for the install instructions)

Running MaxQuant

>$ mono MaxQuant/bin/MaxQuantCmd.exe mqpar.xml

The default output options aren’t too descriptive to the exact task that MaxQuant is busy with, especially during the “Preparing for searches” step. You can query the exact status of your search by printing and grepping the process file names.

>$ ls $MQ_combined_folder/combined/proc | grep started

MSMS_first_search 16.started.txt
MSMS_first_search 26.started.txt
MSMS_first_search 36.started.txt
MSMS_first_search 46.started.txt

You can also print the #runningTimes.txt table to see exactly which steps are taking the longest

>$ cat $MQ_combined_folder/combined/proc/#runningTimes.txt

Job Running time [min]  Start date  Start time  End date  End time
Configuring   0.0168226333333333  14/11/2018  11:03:03  14/11/2018  11:03:04
Testing fasta files   0.0334432333333333  14/11/2018  11:03:04  14/11/2018  11:03:06
Testing raw files   0.11721345  14/11/2018  11:03:06  14/11/2018  11:03:13
Feature detection   5.0676712 14/11/2018  11:03:13  14/11/2018  11:08:17
Tmp folder cleanup  0.0172500333333333  14/11/2018  11:08:17  14/11/2018  11:08:18
Calculating peak properties   3.06749265  14/11/2018  11:08:18  14/11/2018  11:11:22
Combining apl files for first search  0.216795733333333 14/11/2018  11:11:22  14/11/2018  11:11:35
Preparing searches  28.8861353833333  14/11/2018  11:11:35  14/11/2018  11:40:28
MS/MS first search  68.32271715 14/11/2018  11:40:28  14/11/2018  12:48:47
...

Partial Processing

The partial processing feature of the MaxQuant GUI is present in the command-line interface, but to run it you first need to list out the individual steps that MaxQuant wants to take. You can list these out by using the “dry-run” command – appending -n to the normal execution command.

>$ mono MaxQuant/bin/MaxQuantCmd.exe mqpar.xml -n

id  number of threads job name
0 1 Configuring 
1 1 Testing fasta files 
2 8 Testing raw files 
3 8 Feature detection 

...

39  1 Prepare writing tables  
40  8 Writing tables 
41  1 Finish writing tables

Then use the --partial-processing=[id] flag to tell MaxQuant to start at that specific step

>$ mono MaxQuant/bin/MaxQuantCmd.exe mqpar.xml --partial-processing=35

For the best chance to run partial processing successfully, you should have run the previous steps with the same mqpar.xml configuration file. Although it could be possible that partial processing tolerates small configuration changes (for example, the feature detection step may not depend at all on the variable modification settings), for reproducibility’s sake it is best to be consistent with your configuration files. (I haven’t tested this thoroughly, so if you know more get back to me!)

Helper Scripts

Nowadays I’m running MaxQuant on Northeastern’s Research Computing cluster which is hosted at the Massachusetts Green High-Performance Computing Center (MGHPCC) in Holyoke, MA. It uses the Slurm workload manager to run tasks, and as such I have to package all of my MaxQuant calls into slurm-compatible bash scripts.

All code is hosted in this GitHub repo

I have one python script, gen_mqpar.py, which takes as input the mqpar.xml template, folder(s) of raw files, and some other parameters to generate both a filled-out mqpar.xml file and a bash script that can be run by slurm.

>$ ./gen_mqpar.py -h
usage: gen_mqpar.py [-h] -o OUTFILE [-mq {1_6_2_3,1_6_2_10,1_6_3_2,1_6_3_3}]
                    [-t THREADS]
                    input_xml raw_file_folders [raw_file_folders ...]

MaxQuant for linux

positional arguments:
  input_xml
  raw_file_folders

optional arguments:
  -h, --help            show this help message and exit
  -o OUTFILE, --outfile OUTFILE
  -mq {1_6_2_3,1_6_2_10,1_6_3_2,1_6_3_3}, --mq-version {1_6_2_3,1_6_2_10,1_6_3_2,1_6_3_3}
                        MaxQuant version
  -t THREADS, --threads THREADS
                        Number of threads to specify in MaxQuant configuration
                        file

Example:

>$ ./gen_mqpar.py templates/SILAC.xml raw_files/FP93 -o fp93_silac.xml -t 6
success!

I keep a list of templates in a folder, templates/, which I can pull from when I want to change the parameters of a search. For example, I might do a normal TMT search to get quantitation, and also search where TMT is a variable mod to see my labeling efficiency.

Also note that I keep multiple versions of MaxQuant on hand, in case any of them breaks for some reason or another. I manage these versions with aliases in my .bashrc:

MQ_1_6_2_3=~/MQ_1.6.2.3/bin/MaxQuantCmd.exe
MQ_1_6_2_10=~/MQ_1.6.2.10/bin/MaxQuantCmd.exe
MQ_1_6_3_2=~/MQ_1.6.3.2/bin/MaxQuantCmd.exe
MQ_1_6_3_3=~/MQ_1.6.3.3/bin/MaxQuantCmd.exe

As I want all of my MaxQuant versions to have the same modifications.xml list, I have a symlink going from bin/conf/modifications.xml to a common modifications.xml file that I maintain.

The script outputs an XML file that is ready to run by MaxQuantCmd.exe, and a shell script that can be queued up by the slurm job manager:

#!/bin/sh
#SBATCH --job-name=fp103f
#SBATCH --output=fp103f.out
#SBATCH --ntasks=1
#SBATCH --cpus-per-task=16
#SBATCH --mem-per-cpu=1000
#SBATCH --time=24:0:0
#SBATCH --partition=general

source /home/chen.alb/.bashrc
srun mono $MQ_1_6_3_3 /home/chen.alb/fp103f.xml

Check out the git repo with all the code and some examples shown: https://github.com/atc3/maxquant_linux_guide.

All in all, this workflow is great for me, as I just upload some raw files, then run:

>$ ./gen_mqpar.py templates/SILAC.xml raw_files/FP93 -o fp93_silac.xml -t 6
>$ sbatch scripts/fp93_silac.sh
...

And I’m done! I have to wait for a long time for the run to finish, of course, but I try to not babysit the task and just check back after a few hours.

Notes:

The gen_mqpar.py script does create separate output, temp, and andromeda index directories for each search. This was done because I noticed that sharing any of these directories could result in conflicts where two concurrent searches would try to access the same resource at the same time and then one of them would crash. I know this does slow down my search times, especially since all the andromeda indices are separate, but the cluster is fast anyways so I’m willing to take this hit.

Apologies to ∆

Bonus: MIA AIM: