Code

public void adjustAllSelfLoops()
{
    for (DiagramNode node : diagram.getNodes())
    {
        adjustSelfLoop(node);
    }
}

public void adjustSelfLoop(DiagramNode node)
{
    AnchorPattern ap = node.getAnchorPattern();
    if (ap == null || ap.getPoints().size() == 0)
        return;

    int numPoints = ap.getPoints().size();
    boolean[] occupied = new boolean[numPoints];

    ArrayList<DiagramLink> selfLoops = new ArrayList<>();
    for (DiagramLink link : node.getOutgoingLinks())
    {
        if (link.getDestination() == node)
        {
            selfLoops.add(link);
        }
        else
        {
            // Outgoing non-self-loop
            int anchor = link.getOriginAnchor();
            if (anchor >= 0 && anchor < numPoints)
            {
                occupied[anchor] = true;
            }
            else
            {
                int closest = getClosestAnchorIndex(node, link.getStartPoint());
                if (closest >= 0 && closest < numPoints)
                    occupied[closest] = true;
            }
        }
    }

    for (DiagramLink link : node.getIncomingLinks())
    {
        if (link.getOrigin() != node)
        {
            // Incoming non-self-loop
            int anchor = link.getDestinationAnchor();
            if (anchor >= 0 && anchor < numPoints)
            {
                occupied[anchor] = true;
            }
            else
            {
                int closest = getClosestAnchorIndex(node, link.getEndPoint());
                if (closest >= 0 && closest < numPoints)
                    occupied[closest] = true;
            }
        }
    }

    if (selfLoops.isEmpty())
        return;

    // Move each self-loop to an unoccupied anchor point
    int searchIdx = 0;
    for (DiagramLink selfLoop : selfLoops)
    {
        int freeIdx = -1;
        for (int i = 0; i < numPoints; i++)
        {
            int candidate = (searchIdx + i) % numPoints;
            if (!occupied[candidate])
            {
                freeIdx = candidate;
                break;
            }
        }

        if (freeIdx == -1)
            break;

        occupied[freeIdx] = true;
        // advance search index so subsequent self-loops pick different points
        searchIdx = (freeIdx + 4) % numPoints;

        selfLoop.setOriginAnchor(freeIdx);
        selfLoop.setDestinationAnchor(freeIdx);

        selfLoop.setShape(LinkShape.Bezier);
        selfLoop.setSegmentCount(1);

        AnchorPoint anchorPt = ap.getPoints().get(freeIdx);
        Point2D p = getAnchorPointPos(node, anchorPt);
        float px = (float)p.getX();
        float py = (float)p.getY();

        float loopDist = 14.0f;
        float spread = 9.0f;

        Point2D.Float cp1, cp2;

        // Determine node side associated with the anchor point
        if (anchorPt.getY() <= 0.001f)
        {
            // Top side: curve upwards, middle control points horizontal (parallel to top side)
            cp1 = new Point2D.Float(px - spread, py - loopDist);
            cp2 = new Point2D.Float(px + spread, py - loopDist);
        }
        else if (anchorPt.getX() >= 99.999f)
        {
            // Right side: curve rightwards, middle control points vertical (parallel to right side)
            cp1 = new Point2D.Float(px + loopDist, py - spread);
            cp2 = new Point2D.Float(px + loopDist, py + spread);
        }
        else if (anchorPt.getY() >= 99.999f)
        {
            // Bottom side: curve downwards, middle control points horizontal (parallel to bottom side)
            cp1 = new Point2D.Float(px - spread, py + loopDist);
            cp2 = new Point2D.Float(px + spread, py + loopDist);
        }
        else
        {
            // Left side: curve leftwards, middle control points vertical (parallel to left side)
            cp1 = new Point2D.Float(px - loopDist, py - spread);
            cp2 = new Point2D.Float(px - loopDist, py + spread);
        }

        PointList points = selfLoop.getControlPoints();
        if (points.size() >= 4)
        {
            points.set(0, new Point2D.Float(px, py));
            points.set(1, cp1);
            points.set(2, cp2);
            points.set(3, new Point2D.Float(px, py));
            selfLoop.updateFromPoints();
        }
    }
} 

Code

Point2D getAnchorPointPos(DiagramNode node, AnchorPoint ap)
{
	Rectangle2D r = node.getBounds();
	double x = r.getX() + r.getWidth() * (ap.getX() / 100.0);
	double y = r.getY() + r.getHeight() * (ap.getY() / 100.0);
	return new Point2D.Double(x, y);
}

int getClosestAnchorIndex(DiagramNode node, Point2D pt)
{
	AnchorPattern ap = node.getAnchorPattern();
	if (ap == null || ap.getPoints().size() == 0 || pt == null)
		return -1;

	int bestIdx = -1;
	double bestDist = Double.MAX_VALUE;

	for (int i = 0; i < ap.getPoints().size(); i++)
	{
		AnchorPoint p = ap.getPoints().get(i);
		Point2D pos = getAnchorPointPos(node, p);
		double dist = pos.distance(pt);
		if (dist < bestDist)
		{
			bestDist = dist;
			bestIdx = i;
		}
	}
	return bestIdx;
}